2024-04-22T22:13:15.103 INFO:root:teuthology version: 0.0.1.dev269+g9481b1d 2024-04-22T22:13:15.103 DEBUG:teuthology.run:Teuthology command: teuthology --description orch/cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/repo_digest 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} --name teuthology-2024-04-22_21:08:03-orch-squid-distro-default-smithi --verbose --owner scheduled_teuthology@teuthology --archive /home/teuthworker/archive/teuthology-2024-04-22_21:08:03-orch-squid-distro-default-smithi/7668863 -- /home/teuthworker/archive/teuthology-2024-04-22_21:08:03-orch-squid-distro-default-smithi/7668863/orig.config.yaml 2024-04-22T22:13:15.153 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-22T22:13:15.240 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-04-22_21:08:03-orch-squid-distro-default-smithi/7668863 branch: squid description: orch/cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/repo_digest 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} email: ceph-qa@ceph.com first_in_suite: false job_id: '7668863' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: teuthology-2024-04-22_21:08:03-orch-squid-distro-default-smithi no_nested_subset: false nuke-on-error: true os_type: centos os_version: 9.stream overrides: admin_socket: branch: squid ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mgr/cephadm/use_agent: false mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) log-only-match: - CEPHADM_ sha1: 430e09df97c8fc7dc2b2ae424f68ed11366c540f ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 430e09df97c8fc7dc2b2ae424f68ed11366c540f workunit: branch: squid sha1: c66b8bf2efd3f3988ac1851474c2f98eb2ca30d9 owner: scheduled_teuthology@teuthology priority: 100 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b seed: 207 sha1: 430e09df97c8fc7dc2b2ae424f68ed11366c540f sleep_before_teardown: 0 subset: 24/64 suite: orch suite_branch: squid suite_path: /home/teuthworker/src/git.ceph.com_ceph_c66b8bf2efd3f3988ac1851474c2f98eb2ca30d9/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: c66b8bf2efd3f3988ac1851474c2f98eb2ca30d9 targets: smithi067.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKtUaSojXmXB8EKXCcQdK+MdcQRTSMFEMXHra93PmKHxhrNISkr9m2nCZkftJhuIK2PX26IMVeIRLb1OWg1BSZE= smithi086.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKzmqenF1jgMllBMhNcGRN1q0AhIAk99xkSim47SIa60lF4VTPqScAaVnw18Sr/mbRC7Ald58Ha35xjeWVHAlxM= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 - cephadm.shell: mon.a: - ceph config set mgr mgr/cephadm/use_repo_digest false --force - cephadm.shell: env: - sha1 mon.a: - radosgw-admin realm create --rgw-realm=r --default - radosgw-admin zonegroup create --rgw-zonegroup=default --master --default - radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default - radosgw-admin period update --rgw-realm=r --commit - ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000 - ceph osd pool create foo - rbd pool init foo - ceph orch apply iscsi foo u p - sleep 180 - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph orch ps - ceph versions - ceph -s - ceph orch ls - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 2' - ceph mgr fail - sleep 180 - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph health detail - ceph -s - ceph mgr fail - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 1' - ceph mgr fail - sleep 180 - ceph orch ps - ceph orch ls - ceph versions - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph versions | jq -e '.mgr | length == 1' - ceph versions | jq -e '.mgr | keys' | grep $sha1 - ceph versions | jq -e '.overall | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 1' - ceph versions | jq -e '.mon | keys' | grep $sha1 - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 5' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 7' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 8' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 1' - ceph versions | jq -e '.osd | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.rgw | length == 1' - ceph versions | jq -e '.rgw | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 mon.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph versions - ceph orch upgrade status - ceph health detail - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - ceph orch ls | grep '^osd ' - cephadm.shell: mon.a: - ceph orch upgrade ls - ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0 - ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2 teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17 timestamp: 2024-04-22_21:08:03 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-04-22T22:13:15.240 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_c66b8bf2efd3f3988ac1851474c2f98eb2ca30d9/qa; will attempt to use it 2024-04-22T22:13:15.241 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_c66b8bf2efd3f3988ac1851474c2f98eb2ca30d9/qa/tasks 2024-04-22T22:13:15.241 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-22T22:13:15.243 INFO:teuthology.task.internal:Checking packages... 2024-04-22T22:13:15.267 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '430e09df97c8fc7dc2b2ae424f68ed11366c540f' 2024-04-22T22:13:15.268 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-22T22:13:15.268 INFO:teuthology.packaging:ref: None 2024-04-22T22:13:15.268 INFO:teuthology.packaging:tag: None 2024-04-22T22:13:15.268 INFO:teuthology.packaging:branch: squid 2024-04-22T22:13:15.268 INFO:teuthology.packaging:sha1: 430e09df97c8fc7dc2b2ae424f68ed11366c540f 2024-04-22T22:13:15.268 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid 2024-04-22T22:13:15.732 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-2420.g246d7127 2024-04-22T22:13:15.733 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-22T22:13:15.740 INFO:teuthology.task.internal:no buildpackages task found 2024-04-22T22:13:15.740 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-22T22:13:15.749 INFO:teuthology.task.internal:Saving configuration 2024-04-22T22:13:15.773 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-22T22:13:15.779 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-22T22:13:15.806 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi067.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-22_21:08:03-orch-squid-distro-default-smithi/7668863', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-22 22:05:57.209905', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKtUaSojXmXB8EKXCcQdK+MdcQRTSMFEMXHra93PmKHxhrNISkr9m2nCZkftJhuIK2PX26IMVeIRLb1OWg1BSZE='} 2024-04-22T22:13:15.827 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi086.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-22_21:08:03-orch-squid-distro-default-smithi/7668863', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-22 22:05:57.211382', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKzmqenF1jgMllBMhNcGRN1q0AhIAk99xkSim47SIa60lF4VTPqScAaVnw18Sr/mbRC7Ald58Ha35xjeWVHAlxM='} 2024-04-22T22:13:15.827 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-22T22:13:15.834 INFO:teuthology.task.internal:roles: ubuntu@smithi067.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-04-22T22:13:15.834 INFO:teuthology.task.internal:roles: ubuntu@smithi086.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-04-22T22:13:15.834 INFO:teuthology.run_tasks:Running task console_log... 2024-04-22T22:13:15.987 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f5c615990d0>, signals=[15]) 2024-04-22T22:13:15.988 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-22T22:13:15.995 INFO:teuthology.task.internal:Opening connections... 2024-04-22T22:13:15.995 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi067.front.sepia.ceph.com 2024-04-22T22:13:15.996 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi067.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-22T22:13:16.073 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi086.front.sepia.ceph.com 2024-04-22T22:13:16.075 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi086.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-22T22:13:16.154 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-22T22:13:16.161 DEBUG:teuthology.orchestra.run.smithi067:> uname -m 2024-04-22T22:13:16.180 INFO:teuthology.orchestra.run.smithi067.stdout:x86_64 2024-04-22T22:13:16.180 DEBUG:teuthology.orchestra.run.smithi067:> cat /etc/os-release 2024-04-22T22:13:16.238 INFO:teuthology.orchestra.run.smithi067.stdout:NAME="CentOS Stream" 2024-04-22T22:13:16.238 INFO:teuthology.orchestra.run.smithi067.stdout:VERSION="9" 2024-04-22T22:13:16.238 INFO:teuthology.orchestra.run.smithi067.stdout:ID="centos" 2024-04-22T22:13:16.238 INFO:teuthology.orchestra.run.smithi067.stdout:ID_LIKE="rhel fedora" 2024-04-22T22:13:16.238 INFO:teuthology.orchestra.run.smithi067.stdout:VERSION_ID="9" 2024-04-22T22:13:16.238 INFO:teuthology.orchestra.run.smithi067.stdout:PLATFORM_ID="platform:el9" 2024-04-22T22:13:16.238 INFO:teuthology.orchestra.run.smithi067.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-22T22:13:16.238 INFO:teuthology.orchestra.run.smithi067.stdout:ANSI_COLOR="0;31" 2024-04-22T22:13:16.238 INFO:teuthology.orchestra.run.smithi067.stdout:LOGO="fedora-logo-icon" 2024-04-22T22:13:16.238 INFO:teuthology.orchestra.run.smithi067.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-22T22:13:16.238 INFO:teuthology.orchestra.run.smithi067.stdout:HOME_URL="https://centos.org/" 2024-04-22T22:13:16.239 INFO:teuthology.orchestra.run.smithi067.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-22T22:13:16.239 INFO:teuthology.orchestra.run.smithi067.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-22T22:13:16.239 INFO:teuthology.orchestra.run.smithi067.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-22T22:13:16.241 INFO:teuthology.lock.ops:Updating smithi067.front.sepia.ceph.com on lock server 2024-04-22T22:13:16.268 DEBUG:teuthology.orchestra.run.smithi086:> uname -m 2024-04-22T22:13:16.286 INFO:teuthology.orchestra.run.smithi086.stdout:x86_64 2024-04-22T22:13:16.287 DEBUG:teuthology.orchestra.run.smithi086:> cat /etc/os-release 2024-04-22T22:13:16.343 INFO:teuthology.orchestra.run.smithi086.stdout:NAME="CentOS Stream" 2024-04-22T22:13:16.344 INFO:teuthology.orchestra.run.smithi086.stdout:VERSION="9" 2024-04-22T22:13:16.344 INFO:teuthology.orchestra.run.smithi086.stdout:ID="centos" 2024-04-22T22:13:16.344 INFO:teuthology.orchestra.run.smithi086.stdout:ID_LIKE="rhel fedora" 2024-04-22T22:13:16.344 INFO:teuthology.orchestra.run.smithi086.stdout:VERSION_ID="9" 2024-04-22T22:13:16.344 INFO:teuthology.orchestra.run.smithi086.stdout:PLATFORM_ID="platform:el9" 2024-04-22T22:13:16.344 INFO:teuthology.orchestra.run.smithi086.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-22T22:13:16.344 INFO:teuthology.orchestra.run.smithi086.stdout:ANSI_COLOR="0;31" 2024-04-22T22:13:16.344 INFO:teuthology.orchestra.run.smithi086.stdout:LOGO="fedora-logo-icon" 2024-04-22T22:13:16.344 INFO:teuthology.orchestra.run.smithi086.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-22T22:13:16.344 INFO:teuthology.orchestra.run.smithi086.stdout:HOME_URL="https://centos.org/" 2024-04-22T22:13:16.344 INFO:teuthology.orchestra.run.smithi086.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-22T22:13:16.344 INFO:teuthology.orchestra.run.smithi086.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-22T22:13:16.344 INFO:teuthology.orchestra.run.smithi086.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-22T22:13:16.345 INFO:teuthology.lock.ops:Updating smithi086.front.sepia.ceph.com on lock server 2024-04-22T22:13:16.368 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-22T22:13:16.377 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-22T22:13:16.383 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-22T22:13:16.383 DEBUG:teuthology.orchestra.run.smithi067:> test '!' -e /home/ubuntu/cephtest 2024-04-22T22:13:16.386 DEBUG:teuthology.orchestra.run.smithi086:> test '!' -e /home/ubuntu/cephtest 2024-04-22T22:13:16.403 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-22T22:13:16.410 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-22T22:13:16.410 DEBUG:teuthology.orchestra.run.smithi067:> test -z $(ls -A /var/lib/ceph) 2024-04-22T22:13:16.443 DEBUG:teuthology.orchestra.run.smithi086:> test -z $(ls -A /var/lib/ceph) 2024-04-22T22:13:16.469 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-22T22:13:16.522 INFO:teuthology.run_tasks:Running task kernel... 2024-04-22T22:13:16.536 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-22T22:13:16.536 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-22T22:13:16.537 DEBUG:teuthology.orchestra.run.smithi067:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-22T22:13:16.537 DEBUG:teuthology.orchestra.run.smithi086:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-22T22:13:16.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T22:13:16.553 DEBUG:teuthology.orchestra.run.smithi086:> uname -r 2024-04-22T22:13:16.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T22:13:16.554 DEBUG:teuthology.orchestra.run.smithi067:> uname -r 2024-04-22T22:13:16.611 INFO:teuthology.orchestra.run.smithi086.stdout:5.14.0-437.el9.x86_64 2024-04-22T22:13:16.611 INFO:teuthology.task.kernel:Running kernel on smithi086: 5.14.0-437.el9.x86_64 2024-04-22T22:13:16.612 DEBUG:teuthology.orchestra.run.smithi086:> sudo yum install -y kernel 2024-04-22T22:13:16.613 INFO:teuthology.orchestra.run.smithi067.stdout:5.14.0-437.el9.x86_64 2024-04-22T22:13:16.613 INFO:teuthology.task.kernel:Running kernel on smithi067: 5.14.0-437.el9.x86_64 2024-04-22T22:13:16.613 DEBUG:teuthology.orchestra.run.smithi067:> sudo yum install -y kernel 2024-04-22T22:13:18.931 INFO:teuthology.orchestra.run.smithi086.stdout:CentOS Stream 9 - BaseOS 11 MB/s | 8.0 MB 00:00 2024-04-22T22:13:20.523 INFO:teuthology.orchestra.run.smithi067.stdout:CentOS Stream 9 - BaseOS 3.4 MB/s | 8.0 MB 00:02 2024-04-22T22:13:21.040 INFO:teuthology.orchestra.run.smithi086.stdout:CentOS Stream 9 - AppStream 18 MB/s | 19 MB 00:01 2024-04-22T22:13:22.757 INFO:teuthology.orchestra.run.smithi067.stdout:CentOS Stream 9 - AppStream 17 MB/s | 19 MB 00:01 2024-04-22T22:13:26.166 INFO:teuthology.orchestra.run.smithi086.stdout:CentOS Stream 9 - CRB 4.2 MB/s | 6.2 MB 00:01 2024-04-22T22:13:28.477 INFO:teuthology.orchestra.run.smithi067.stdout:CentOS Stream 9 - CRB 3.1 MB/s | 6.2 MB 00:02 2024-04-22T22:13:28.916 INFO:teuthology.orchestra.run.smithi086.stdout:CentOS Stream 9 - Extras packages 11 kB/s | 16 kB 00:01 2024-04-22T22:13:29.933 INFO:teuthology.orchestra.run.smithi067.stdout:CentOS Stream 9 - Extras packages 69 kB/s | 16 kB 00:00 2024-04-22T22:13:31.170 INFO:teuthology.orchestra.run.smithi086.stdout:Extra Packages for Enterprise Linux 9.8 MB/s | 21 MB 00:02 2024-04-22T22:13:32.351 INFO:teuthology.orchestra.run.smithi067.stdout:Extra Packages for Enterprise Linux 9.1 MB/s | 21 MB 00:02 2024-04-22T22:13:36.168 INFO:teuthology.orchestra.run.smithi086.stdout:lab-extras 13 kB/s | 1.7 kB 00:00 2024-04-22T22:13:37.585 INFO:teuthology.orchestra.run.smithi067.stdout:lab-extras 33 kB/s | 1.7 kB 00:00 2024-04-22T22:13:38.032 INFO:teuthology.orchestra.run.smithi086.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-22T22:13:38.032 INFO:teuthology.orchestra.run.smithi086.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-04-22T22:13:38.121 INFO:teuthology.orchestra.run.smithi086.stdout:Dependencies resolved. 2024-04-22T22:13:38.122 INFO:teuthology.orchestra.run.smithi086.stdout:Nothing to do. 2024-04-22T22:13:38.122 INFO:teuthology.orchestra.run.smithi086.stdout:Complete! 2024-04-22T22:13:38.222 DEBUG:teuthology.orchestra.run.smithi086:> echo no | sudo yum reinstall kernel || true 2024-04-22T22:13:38.722 INFO:teuthology.orchestra.run.smithi086.stdout:Last metadata expiration check: 0:00:02 ago on Mon 22 Apr 2024 10:13:36 PM UTC. 2024-04-22T22:13:38.841 INFO:teuthology.orchestra.run.smithi086.stdout:Dependencies resolved. 2024-04-22T22:13:38.855 INFO:teuthology.orchestra.run.smithi086.stdout:================================================================================ 2024-04-22T22:13:38.855 INFO:teuthology.orchestra.run.smithi086.stdout: Package Architecture Version Repository Size 2024-04-22T22:13:38.855 INFO:teuthology.orchestra.run.smithi086.stdout:================================================================================ 2024-04-22T22:13:38.856 INFO:teuthology.orchestra.run.smithi086.stdout:Reinstalling: 2024-04-22T22:13:38.856 INFO:teuthology.orchestra.run.smithi086.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-22T22:13:38.856 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-22T22:13:38.856 INFO:teuthology.orchestra.run.smithi086.stdout:Transaction Summary 2024-04-22T22:13:38.856 INFO:teuthology.orchestra.run.smithi086.stdout:================================================================================ 2024-04-22T22:13:38.856 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-22T22:13:38.856 INFO:teuthology.orchestra.run.smithi086.stdout:Total download size: 255 k 2024-04-22T22:13:38.856 INFO:teuthology.orchestra.run.smithi086.stdout:Installed size: 0 2024-04-22T22:13:38.856 INFO:teuthology.orchestra.run.smithi086.stderr:Operation aborted. 2024-04-22T22:13:38.917 INFO:teuthology.orchestra.run.smithi086.stdout:Is this ok [y/N]: 2024-04-22T22:13:38.917 DEBUG:teuthology.orchestra.run.smithi086:> sudo yum reinstall -y kernel || true 2024-04-22T22:13:39.412 INFO:teuthology.orchestra.run.smithi086.stdout:Last metadata expiration check: 0:00:03 ago on Mon 22 Apr 2024 10:13:36 PM UTC. 2024-04-22T22:13:39.525 INFO:teuthology.orchestra.run.smithi067.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-22T22:13:39.525 INFO:teuthology.orchestra.run.smithi067.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-04-22T22:13:39.527 INFO:teuthology.orchestra.run.smithi086.stdout:Dependencies resolved. 2024-04-22T22:13:39.528 INFO:teuthology.orchestra.run.smithi086.stdout:================================================================================ 2024-04-22T22:13:39.528 INFO:teuthology.orchestra.run.smithi086.stdout: Package Architecture Version Repository Size 2024-04-22T22:13:39.529 INFO:teuthology.orchestra.run.smithi086.stdout:================================================================================ 2024-04-22T22:13:39.529 INFO:teuthology.orchestra.run.smithi086.stdout:Reinstalling: 2024-04-22T22:13:39.529 INFO:teuthology.orchestra.run.smithi086.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-22T22:13:39.529 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-22T22:13:39.529 INFO:teuthology.orchestra.run.smithi086.stdout:Transaction Summary 2024-04-22T22:13:39.529 INFO:teuthology.orchestra.run.smithi086.stdout:================================================================================ 2024-04-22T22:13:39.529 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-22T22:13:39.529 INFO:teuthology.orchestra.run.smithi086.stdout:Total download size: 255 k 2024-04-22T22:13:39.529 INFO:teuthology.orchestra.run.smithi086.stdout:Installed size: 0 2024-04-22T22:13:39.529 INFO:teuthology.orchestra.run.smithi086.stdout:Downloading Packages: 2024-04-22T22:13:39.615 INFO:teuthology.orchestra.run.smithi067.stdout:Dependencies resolved. 2024-04-22T22:13:39.616 INFO:teuthology.orchestra.run.smithi067.stdout:Nothing to do. 2024-04-22T22:13:39.617 INFO:teuthology.orchestra.run.smithi067.stdout:Complete! 2024-04-22T22:13:39.726 DEBUG:teuthology.orchestra.run.smithi067:> echo no | sudo yum reinstall kernel || true 2024-04-22T22:13:40.243 INFO:teuthology.orchestra.run.smithi067.stdout:Last metadata expiration check: 0:00:03 ago on Mon 22 Apr 2024 10:13:37 PM UTC. 2024-04-22T22:13:40.374 INFO:teuthology.orchestra.run.smithi067.stdout:Dependencies resolved. 2024-04-22T22:13:40.386 INFO:teuthology.orchestra.run.smithi067.stdout:================================================================================ 2024-04-22T22:13:40.387 INFO:teuthology.orchestra.run.smithi067.stdout: Package Architecture Version Repository Size 2024-04-22T22:13:40.387 INFO:teuthology.orchestra.run.smithi067.stdout:================================================================================ 2024-04-22T22:13:40.387 INFO:teuthology.orchestra.run.smithi067.stdout:Reinstalling: 2024-04-22T22:13:40.387 INFO:teuthology.orchestra.run.smithi067.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-22T22:13:40.387 INFO:teuthology.orchestra.run.smithi067.stdout: 2024-04-22T22:13:40.387 INFO:teuthology.orchestra.run.smithi067.stdout:Transaction Summary 2024-04-22T22:13:40.387 INFO:teuthology.orchestra.run.smithi067.stdout:================================================================================ 2024-04-22T22:13:40.387 INFO:teuthology.orchestra.run.smithi067.stdout: 2024-04-22T22:13:40.387 INFO:teuthology.orchestra.run.smithi067.stdout:Total download size: 255 k 2024-04-22T22:13:40.387 INFO:teuthology.orchestra.run.smithi067.stdout:Installed size: 0 2024-04-22T22:13:40.387 INFO:teuthology.orchestra.run.smithi067.stderr:Operation aborted. 2024-04-22T22:13:40.448 INFO:teuthology.orchestra.run.smithi067.stdout:Is this ok [y/N]: 2024-04-22T22:13:40.448 DEBUG:teuthology.orchestra.run.smithi067:> sudo yum reinstall -y kernel || true 2024-04-22T22:13:40.528 INFO:teuthology.orchestra.run.smithi086.stdout:kernel-5.14.0-437.el9.x86_64.rpm 274 kB/s | 255 kB 00:00 2024-04-22T22:13:40.529 INFO:teuthology.orchestra.run.smithi086.stdout:-------------------------------------------------------------------------------- 2024-04-22T22:13:40.529 INFO:teuthology.orchestra.run.smithi086.stdout:Total 255 kB/s | 255 kB 00:00 2024-04-22T22:13:40.552 INFO:teuthology.orchestra.run.smithi086.stdout:Running transaction check 2024-04-22T22:13:40.722 INFO:teuthology.orchestra.run.smithi086.stdout:Transaction check succeeded. 2024-04-22T22:13:40.723 INFO:teuthology.orchestra.run.smithi086.stdout:Running transaction test 2024-04-22T22:13:40.728 INFO:teuthology.orchestra.run.smithi086.stdout:Transaction test succeeded. 2024-04-22T22:13:40.728 INFO:teuthology.orchestra.run.smithi086.stdout:Running transaction 2024-04-22T22:13:40.885 INFO:teuthology.orchestra.run.smithi086.stdout: Preparing : 1/1 2024-04-22T22:13:40.962 INFO:teuthology.orchestra.run.smithi086.stdout: Reinstalling : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-22T22:13:40.962 INFO:teuthology.orchestra.run.smithi086.stdout: Cleanup : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-22T22:13:40.963 INFO:teuthology.orchestra.run.smithi067.stdout:Last metadata expiration check: 0:00:03 ago on Mon 22 Apr 2024 10:13:37 PM UTC. 2024-04-22T22:13:41.085 INFO:teuthology.orchestra.run.smithi067.stdout:Dependencies resolved. 2024-04-22T22:13:41.086 INFO:teuthology.orchestra.run.smithi067.stdout:================================================================================ 2024-04-22T22:13:41.086 INFO:teuthology.orchestra.run.smithi067.stdout: Package Architecture Version Repository Size 2024-04-22T22:13:41.086 INFO:teuthology.orchestra.run.smithi067.stdout:================================================================================ 2024-04-22T22:13:41.086 INFO:teuthology.orchestra.run.smithi067.stdout:Reinstalling: 2024-04-22T22:13:41.086 INFO:teuthology.orchestra.run.smithi067.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-22T22:13:41.086 INFO:teuthology.orchestra.run.smithi067.stdout: 2024-04-22T22:13:41.086 INFO:teuthology.orchestra.run.smithi067.stdout:Transaction Summary 2024-04-22T22:13:41.086 INFO:teuthology.orchestra.run.smithi067.stdout:================================================================================ 2024-04-22T22:13:41.086 INFO:teuthology.orchestra.run.smithi067.stdout: 2024-04-22T22:13:41.086 INFO:teuthology.orchestra.run.smithi067.stdout:Total download size: 255 k 2024-04-22T22:13:41.086 INFO:teuthology.orchestra.run.smithi067.stdout:Installed size: 0 2024-04-22T22:13:41.086 INFO:teuthology.orchestra.run.smithi067.stdout:Downloading Packages: 2024-04-22T22:13:41.445 INFO:teuthology.orchestra.run.smithi067.stdout:kernel-5.14.0-437.el9.x86_64.rpm 1.1 MB/s | 255 kB 00:00 2024-04-22T22:13:41.445 INFO:teuthology.orchestra.run.smithi067.stdout:-------------------------------------------------------------------------------- 2024-04-22T22:13:41.446 INFO:teuthology.orchestra.run.smithi067.stdout:Total 709 kB/s | 255 kB 00:00 2024-04-22T22:13:41.468 INFO:teuthology.orchestra.run.smithi067.stdout:Running transaction check 2024-04-22T22:13:41.634 INFO:teuthology.orchestra.run.smithi086.stdout: Running scriptlet: kernel-5.14.0-437.el9.x86_64 2/2 2024-04-22T22:13:41.634 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-22T22:13:41.659 INFO:teuthology.orchestra.run.smithi067.stdout:Transaction check succeeded. 2024-04-22T22:13:41.659 INFO:teuthology.orchestra.run.smithi067.stdout:Running transaction test 2024-04-22T22:13:41.664 INFO:teuthology.orchestra.run.smithi067.stdout:Transaction test succeeded. 2024-04-22T22:13:41.664 INFO:teuthology.orchestra.run.smithi067.stdout:Running transaction 2024-04-22T22:13:41.819 INFO:teuthology.orchestra.run.smithi067.stdout: Preparing : 1/1 2024-04-22T22:13:41.888 INFO:teuthology.orchestra.run.smithi067.stdout: Reinstalling : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-22T22:13:41.888 INFO:teuthology.orchestra.run.smithi067.stdout: Cleanup : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-22T22:13:42.289 INFO:teuthology.orchestra.run.smithi086.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-22T22:13:42.289 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-22T22:13:42.290 INFO:teuthology.orchestra.run.smithi086.stdout:Reinstalled: 2024-04-22T22:13:42.290 INFO:teuthology.orchestra.run.smithi086.stdout: kernel-5.14.0-437.el9.x86_64 2024-04-22T22:13:42.290 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-22T22:13:42.290 INFO:teuthology.orchestra.run.smithi086.stdout:Complete! 2024-04-22T22:13:42.456 DEBUG:teuthology.orchestra.run.smithi086:> rpm -q kernel | sort -rV | head -n 1 2024-04-22T22:13:42.503 INFO:teuthology.orchestra.run.smithi086.stdout:kernel-5.14.0-437.el9.x86_64 2024-04-22T22:13:42.504 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-437.el9.x86_64 2024-04-22T22:13:42.504 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-22T22:13:42.504 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-04-22T22:13:42.504 DEBUG:teuthology.orchestra.run.smithi086:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-22T22:13:42.532 INFO:teuthology.orchestra.run.smithi086.stdout:ttyS1 2024-04-22T22:13:42.556 DEBUG:teuthology.parallel:result is None 2024-04-22T22:13:42.612 INFO:teuthology.orchestra.run.smithi067.stdout: Running scriptlet: kernel-5.14.0-437.el9.x86_64 2/2 2024-04-22T22:13:42.613 INFO:teuthology.orchestra.run.smithi067.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-22T22:13:43.361 INFO:teuthology.orchestra.run.smithi067.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-22T22:13:43.361 INFO:teuthology.orchestra.run.smithi067.stdout: 2024-04-22T22:13:43.361 INFO:teuthology.orchestra.run.smithi067.stdout:Reinstalled: 2024-04-22T22:13:43.361 INFO:teuthology.orchestra.run.smithi067.stdout: kernel-5.14.0-437.el9.x86_64 2024-04-22T22:13:43.361 INFO:teuthology.orchestra.run.smithi067.stdout: 2024-04-22T22:13:43.361 INFO:teuthology.orchestra.run.smithi067.stdout:Complete! 2024-04-22T22:13:43.529 DEBUG:teuthology.orchestra.run.smithi067:> rpm -q kernel | sort -rV | head -n 1 2024-04-22T22:13:43.618 INFO:teuthology.orchestra.run.smithi067.stdout:kernel-5.14.0-437.el9.x86_64 2024-04-22T22:13:43.619 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-437.el9.x86_64 2024-04-22T22:13:43.619 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-22T22:13:43.619 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-04-22T22:13:43.619 DEBUG:teuthology.orchestra.run.smithi067:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-22T22:13:43.648 INFO:teuthology.orchestra.run.smithi067.stdout:ttyS1 2024-04-22T22:13:43.670 DEBUG:teuthology.parallel:result is None 2024-04-22T22:13:43.670 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-22T22:13:43.678 INFO:teuthology.task.internal:Creating test directory... 2024-04-22T22:13:43.678 DEBUG:teuthology.orchestra.run.smithi067:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-22T22:13:43.712 DEBUG:teuthology.orchestra.run.smithi086:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-22T22:13:43.732 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-22T22:13:43.740 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-22T22:13:43.746 INFO:teuthology.task.internal:Creating archive directory... 2024-04-22T22:13:43.747 DEBUG:teuthology.orchestra.run.smithi067:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-22T22:13:43.768 DEBUG:teuthology.orchestra.run.smithi086:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-22T22:13:43.798 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-22T22:13:43.805 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-22T22:13:43.805 DEBUG:teuthology.orchestra.run.smithi067:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-22T22:13:43.827 DEBUG:teuthology.orchestra.run.smithi086:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-22T22:13:43.870 INFO:teuthology.orchestra.run.smithi086.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-22T22:13:43.881 INFO:teuthology.orchestra.run.smithi086.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-22T22:13:43.883 INFO:teuthology.orchestra.run.smithi067.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-22T22:13:43.893 INFO:teuthology.orchestra.run.smithi067.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-22T22:13:43.895 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-22T22:13:43.902 INFO:teuthology.task.internal:Configuring sudo... 2024-04-22T22:13:43.903 DEBUG:teuthology.orchestra.run.smithi067:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-22T22:13:43.905 DEBUG:teuthology.orchestra.run.smithi086:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-22T22:13:43.949 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-22T22:13:43.964 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-22T22:13:43.965 DEBUG:teuthology.orchestra.run.smithi067:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-22T22:13:43.969 DEBUG:teuthology.orchestra.run.smithi086:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-22T22:13:44.006 DEBUG:teuthology.orchestra.run.smithi067:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-22T22:13:44.066 DEBUG:teuthology.orchestra.run.smithi067:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-22T22:13:44.108 DEBUG:teuthology.orchestra.run.smithi067:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-22T22:13:44.191 DEBUG:teuthology.orchestra.run.smithi067:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-22T22:13:44.229 DEBUG:teuthology.orchestra.run.smithi067:> set -ex 2024-04-22T22:13:44.229 DEBUG:teuthology.orchestra.run.smithi067:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-22T22:13:44.294 DEBUG:teuthology.orchestra.run.smithi086:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-22T22:13:44.343 DEBUG:teuthology.orchestra.run.smithi086:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-22T22:13:44.393 DEBUG:teuthology.orchestra.run.smithi086:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-22T22:13:44.481 DEBUG:teuthology.orchestra.run.smithi086:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-22T22:13:44.523 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-04-22T22:13:44.523 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-22T22:13:44.590 DEBUG:teuthology.orchestra.run.smithi067:> sudo service rsyslog restart 2024-04-22T22:13:44.592 DEBUG:teuthology.orchestra.run.smithi086:> sudo service rsyslog restart 2024-04-22T22:13:44.621 INFO:teuthology.orchestra.run.smithi067.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-22T22:13:44.674 INFO:teuthology.orchestra.run.smithi086.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-22T22:13:45.027 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-22T22:13:45.035 INFO:teuthology.task.internal:Starting timer... 2024-04-22T22:13:45.035 INFO:teuthology.run_tasks:Running task pcp... 2024-04-22T22:13:45.053 INFO:teuthology.run_tasks:Running task selinux... 2024-04-22T22:13:45.063 DEBUG:teuthology.orchestra.run.smithi067:> sudo service auditd rotate 2024-04-22T22:13:45.144 INFO:teuthology.orchestra.run.smithi067.stdout:Rotating logs: 2024-04-22T22:13:45.145 DEBUG:teuthology.orchestra.run.smithi086:> sudo service auditd rotate 2024-04-22T22:13:45.201 INFO:teuthology.orchestra.run.smithi086.stdout:Rotating logs: 2024-04-22T22:13:45.203 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-22T22:13:45.203 DEBUG:teuthology.orchestra.run.smithi067:> /usr/sbin/getenforce 2024-04-22T22:13:45.230 INFO:teuthology.orchestra.run.smithi067.stdout:Permissive 2024-04-22T22:13:45.230 DEBUG:teuthology.orchestra.run.smithi086:> /usr/sbin/getenforce 2024-04-22T22:13:45.248 INFO:teuthology.orchestra.run.smithi086.stdout:Permissive 2024-04-22T22:13:45.248 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi067.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi086.front.sepia.ceph.com': 'permissive'} 2024-04-22T22:13:45.248 DEBUG:teuthology.orchestra.run.smithi067:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-22T22:13:45.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T22:13:45.297 DEBUG:teuthology.orchestra.run.smithi086:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-22T22:13:45.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T22:13:45.322 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-22T22:13:45.323 DEBUG:teuthology.orchestra.run.smithi067:> sudo /usr/sbin/setenforce permissive 2024-04-22T22:13:45.366 DEBUG:teuthology.orchestra.run.smithi086:> sudo /usr/sbin/setenforce permissive 2024-04-22T22:13:45.391 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-22T22:13:45.408 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-22T22:13:45.433 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-22T22:13:45.434 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi067.front.sepia.ceph.com,smithi086.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-22T22:17:27.538 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi067.front.sepia.ceph.com'), Remote(name='ubuntu@smithi086.front.sepia.ceph.com')] 2024-04-22T22:17:27.539 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi067.front.sepia.ceph.com' 2024-04-22T22:17:27.540 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi067.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-22T22:17:27.619 DEBUG:teuthology.orchestra.run.smithi067:> true 2024-04-22T22:17:27.702 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi067.front.sepia.ceph.com' 2024-04-22T22:17:27.703 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi086.front.sepia.ceph.com' 2024-04-22T22:17:27.703 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi086.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-22T22:17:27.777 DEBUG:teuthology.orchestra.run.smithi086:> true 2024-04-22T22:17:27.857 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi086.front.sepia.ceph.com' 2024-04-22T22:17:27.857 INFO:teuthology.run_tasks:Running task clock... 2024-04-22T22:17:27.867 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-04-22T22:17:27.867 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-22T22:17:27.868 DEBUG:teuthology.orchestra.run.smithi067:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-22T22:17:27.870 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-22T22:17:27.870 DEBUG:teuthology.orchestra.run.smithi086:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-22T22:17:27.899 INFO:teuthology.orchestra.run.smithi067.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-22T22:17:27.914 INFO:teuthology.orchestra.run.smithi067.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-22T22:17:27.925 INFO:teuthology.orchestra.run.smithi086.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-22T22:17:27.938 INFO:teuthology.orchestra.run.smithi086.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-22T22:17:27.939 INFO:teuthology.orchestra.run.smithi067.stderr:sudo: ntpd: command not found 2024-04-22T22:17:27.950 INFO:teuthology.orchestra.run.smithi067.stdout:506 Cannot talk to daemon 2024-04-22T22:17:27.960 INFO:teuthology.orchestra.run.smithi086.stderr:sudo: ntpd: command not found 2024-04-22T22:17:27.964 INFO:teuthology.orchestra.run.smithi067.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-22T22:17:27.970 INFO:teuthology.orchestra.run.smithi086.stdout:506 Cannot talk to daemon 2024-04-22T22:17:27.978 INFO:teuthology.orchestra.run.smithi067.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-22T22:17:27.983 INFO:teuthology.orchestra.run.smithi086.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-22T22:17:27.995 INFO:teuthology.orchestra.run.smithi086.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-22T22:17:28.035 INFO:teuthology.orchestra.run.smithi067.stderr:bash: line 1: ntpq: command not found 2024-04-22T22:17:28.039 INFO:teuthology.orchestra.run.smithi067.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-22T22:17:28.039 INFO:teuthology.orchestra.run.smithi067.stdout:=============================================================================== 2024-04-22T22:17:28.040 INFO:teuthology.orchestra.run.smithi067.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T22:17:28.040 INFO:teuthology.orchestra.run.smithi067.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T22:17:28.040 INFO:teuthology.orchestra.run.smithi067.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T22:17:28.040 INFO:teuthology.orchestra.run.smithi067.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T22:17:28.053 INFO:teuthology.orchestra.run.smithi086.stderr:bash: line 1: ntpq: command not found 2024-04-22T22:17:28.055 INFO:teuthology.orchestra.run.smithi086.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-22T22:17:28.055 INFO:teuthology.orchestra.run.smithi086.stdout:=============================================================================== 2024-04-22T22:17:28.055 INFO:teuthology.orchestra.run.smithi086.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T22:17:28.056 INFO:teuthology.orchestra.run.smithi086.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T22:17:28.056 INFO:teuthology.orchestra.run.smithi086.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T22:17:28.056 INFO:teuthology.orchestra.run.smithi086.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T22:17:28.057 INFO:teuthology.run_tasks:Running task cephadm... 2024-04-22T22:17:28.157 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1, 'mgr/cephadm/use_agent': False}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'log-only-match': ['CEPHADM_'], 'sha1': '430e09df97c8fc7dc2b2ae424f68ed11366c540f'} 2024-04-22T22:17:28.157 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-04-22T22:17:28.158 INFO:tasks.cephadm:Cluster fsid is 1ab1a3a8-00f6-11ef-bc93-c7b262605968 2024-04-22T22:17:28.158 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-04-22T22:17:28.158 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.67', 'mon.c': '[v2:172.21.15.67:3301,v1:172.21.15.67:6790]', 'mon.b': '172.21.15.86'} 2024-04-22T22:17:28.158 INFO:tasks.cephadm:First mon is mon.a on smithi067 2024-04-22T22:17:28.158 INFO:tasks.cephadm:First mgr is y 2024-04-22T22:17:28.158 INFO:tasks.cephadm:Normalizing hostnames... 2024-04-22T22:17:28.158 DEBUG:teuthology.orchestra.run.smithi067:> sudo hostname $(hostname -s) 2024-04-22T22:17:28.184 DEBUG:teuthology.orchestra.run.smithi086:> sudo hostname $(hostname -s) 2024-04-22T22:17:28.208 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-04-22T22:17:28.209 DEBUG:teuthology.orchestra.run.smithi067:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-22T22:17:28.405 INFO:teuthology.orchestra.run.smithi067.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Apr 22 22:17 /home/ubuntu/cephtest/cephadm 2024-04-22T22:17:28.405 DEBUG:teuthology.orchestra.run.smithi086:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-22T22:17:28.613 INFO:teuthology.orchestra.run.smithi086.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Apr 22 22:17 /home/ubuntu/cephtest/cephadm 2024-04-22T22:17:28.614 DEBUG:teuthology.orchestra.run.smithi067:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-22T22:17:28.630 DEBUG:teuthology.orchestra.run.smithi086:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-22T22:17:28.657 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-04-22T22:17:28.657 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-04-22T22:17:28.672 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-04-22T22:17:30.184 INFO:teuthology.orchestra.run.smithi067.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-22T22:17:30.305 INFO:teuthology.orchestra.run.smithi086.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-22T22:17:52.972 INFO:teuthology.orchestra.run.smithi086.stdout:{ 2024-04-22T22:17:52.972 INFO:teuthology.orchestra.run.smithi086.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-04-22T22:17:52.973 INFO:teuthology.orchestra.run.smithi086.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-04-22T22:17:52.973 INFO:teuthology.orchestra.run.smithi086.stdout: "repo_digests": [ 2024-04-22T22:17:52.973 INFO:teuthology.orchestra.run.smithi086.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-04-22T22:17:52.973 INFO:teuthology.orchestra.run.smithi086.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-04-22T22:17:52.973 INFO:teuthology.orchestra.run.smithi086.stdout: ] 2024-04-22T22:17:52.973 INFO:teuthology.orchestra.run.smithi086.stdout:} 2024-04-22T22:17:55.374 INFO:teuthology.orchestra.run.smithi067.stdout:{ 2024-04-22T22:17:55.375 INFO:teuthology.orchestra.run.smithi067.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-04-22T22:17:55.375 INFO:teuthology.orchestra.run.smithi067.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-04-22T22:17:55.375 INFO:teuthology.orchestra.run.smithi067.stdout: "repo_digests": [ 2024-04-22T22:17:55.375 INFO:teuthology.orchestra.run.smithi067.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-04-22T22:17:55.375 INFO:teuthology.orchestra.run.smithi067.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-04-22T22:17:55.375 INFO:teuthology.orchestra.run.smithi067.stdout: ] 2024-04-22T22:17:55.375 INFO:teuthology.orchestra.run.smithi067.stdout:} 2024-04-22T22:17:55.399 DEBUG:teuthology.orchestra.run.smithi067:> sudo mkdir -p /etc/ceph 2024-04-22T22:17:55.437 DEBUG:teuthology.orchestra.run.smithi086:> sudo mkdir -p /etc/ceph 2024-04-22T22:17:55.463 DEBUG:teuthology.orchestra.run.smithi067:> sudo chmod 777 /etc/ceph 2024-04-22T22:17:55.504 DEBUG:teuthology.orchestra.run.smithi086:> sudo chmod 777 /etc/ceph 2024-04-22T22:17:55.527 INFO:tasks.cephadm:Writing seed config... 2024-04-22T22:17:55.528 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2024-04-22T22:17:55.528 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-04-22T22:17:55.528 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-04-22T22:17:55.528 INFO:tasks.cephadm: override: [mgr] mgr/cephadm/use_agent = False 2024-04-22T22:17:55.528 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-04-22T22:17:55.528 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-04-22T22:17:55.528 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-04-22T22:17:55.528 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-04-22T22:17:55.528 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-04-22T22:17:55.530 DEBUG:teuthology.orchestra.run.smithi067:> set -ex 2024-04-22T22:17:55.530 DEBUG:teuthology.orchestra.run.smithi067:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-04-22T22:17:55.562 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 1ab1a3a8-00f6-11ef-bc93-c7b262605968 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = true bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 mgr/cephadm/use_agent = False [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-04-22T22:17:55.562 DEBUG:teuthology.orchestra.run.smithi067:mon.a> sudo journalctl -f -n 0 -u ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@mon.a.service 2024-04-22T22:17:55.604 DEBUG:teuthology.orchestra.run.smithi067:mgr.y> sudo journalctl -f -n 0 -u ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@mgr.y.service 2024-04-22T22:17:55.649 INFO:tasks.cephadm:Bootstrapping... 2024-04-22T22:17:55.649 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.67 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-04-22T22:17:55.854 INFO:teuthology.orchestra.run.smithi067.stderr:-------------------------------------------------------------------------------- 2024-04-22T22:17:55.854 INFO:teuthology.orchestra.run.smithi067.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '1ab1a3a8-00f6-11ef-bc93-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.67', '--skip-admin-label'] 2024-04-22T22:17:55.876 INFO:teuthology.orchestra.run.smithi067.stderr:/bin/podman: 4.9.4-dev 2024-04-22T22:17:55.879 INFO:teuthology.orchestra.run.smithi067.stderr:Verifying podman|docker is present... 2024-04-22T22:17:55.987 INFO:teuthology.orchestra.run.smithi067.stderr:/bin/podman: 4.9.4-dev 2024-04-22T22:17:55.989 INFO:teuthology.orchestra.run.smithi067.stderr:Verifying lvm2 is present... 2024-04-22T22:17:55.989 INFO:teuthology.orchestra.run.smithi067.stderr:Verifying time synchronization is in place... 2024-04-22T22:17:55.996 INFO:teuthology.orchestra.run.smithi067.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-22T22:17:56.003 INFO:teuthology.orchestra.run.smithi067.stderr:systemctl: inactive 2024-04-22T22:17:56.010 INFO:teuthology.orchestra.run.smithi067.stderr:systemctl: enabled 2024-04-22T22:17:56.017 INFO:teuthology.orchestra.run.smithi067.stderr:systemctl: active 2024-04-22T22:17:56.017 INFO:teuthology.orchestra.run.smithi067.stderr:Unit chronyd.service is enabled and running 2024-04-22T22:17:56.017 INFO:teuthology.orchestra.run.smithi067.stderr:Repeating the final host check... 2024-04-22T22:17:56.040 INFO:teuthology.orchestra.run.smithi067.stderr:/bin/podman: 4.9.4-dev 2024-04-22T22:17:56.042 INFO:teuthology.orchestra.run.smithi067.stderr:podman (/bin/podman) version 4.9.4 is present 2024-04-22T22:17:56.042 INFO:teuthology.orchestra.run.smithi067.stderr:systemctl is present 2024-04-22T22:17:56.042 INFO:teuthology.orchestra.run.smithi067.stderr:lvcreate is present 2024-04-22T22:17:56.048 INFO:teuthology.orchestra.run.smithi067.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-22T22:17:56.054 INFO:teuthology.orchestra.run.smithi067.stderr:systemctl: inactive 2024-04-22T22:17:56.061 INFO:teuthology.orchestra.run.smithi067.stderr:systemctl: enabled 2024-04-22T22:17:56.068 INFO:teuthology.orchestra.run.smithi067.stderr:systemctl: active 2024-04-22T22:17:56.069 INFO:teuthology.orchestra.run.smithi067.stderr:Unit chronyd.service is enabled and running 2024-04-22T22:17:56.069 INFO:teuthology.orchestra.run.smithi067.stderr:Host looks OK 2024-04-22T22:17:56.069 INFO:teuthology.orchestra.run.smithi067.stderr:Cluster fsid: 1ab1a3a8-00f6-11ef-bc93-c7b262605968 2024-04-22T22:17:56.069 INFO:teuthology.orchestra.run.smithi067.stderr:Acquiring lock 140697448734576 on /run/cephadm/1ab1a3a8-00f6-11ef-bc93-c7b262605968.lock 2024-04-22T22:17:56.069 INFO:teuthology.orchestra.run.smithi067.stderr:Lock 140697448734576 acquired on /run/cephadm/1ab1a3a8-00f6-11ef-bc93-c7b262605968.lock 2024-04-22T22:17:56.069 INFO:teuthology.orchestra.run.smithi067.stderr:Verifying IP 172.21.15.67 port 3300 ... 2024-04-22T22:17:56.070 INFO:teuthology.orchestra.run.smithi067.stderr:Verifying IP 172.21.15.67 port 6789 ... 2024-04-22T22:17:56.070 INFO:teuthology.orchestra.run.smithi067.stderr:Base mon IP is 172.21.15.67, final addrv is [v2:172.21.15.67:3300,v1:172.21.15.67:6789] 2024-04-22T22:17:56.072 INFO:teuthology.orchestra.run.smithi067.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.67 metric 100 2024-04-22T22:17:56.072 INFO:teuthology.orchestra.run.smithi067.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.67 metric 100 2024-04-22T22:17:56.075 INFO:teuthology.orchestra.run.smithi067.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-04-22T22:17:56.075 INFO:teuthology.orchestra.run.smithi067.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 1024 pref medium 2024-04-22T22:17:56.075 INFO:teuthology.orchestra.run.smithi067.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 pref medium 2024-04-22T22:17:56.077 INFO:teuthology.orchestra.run.smithi067.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-04-22T22:17:56.078 INFO:teuthology.orchestra.run.smithi067.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-04-22T22:17:56.078 INFO:teuthology.orchestra.run.smithi067.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-22T22:17:56.078 INFO:teuthology.orchestra.run.smithi067.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-04-22T22:17:56.078 INFO:teuthology.orchestra.run.smithi067.stderr:/sbin/ip: inet6 fe80::8fbb:8197:a5e0:6afc/64 scope link noprefixroute 2024-04-22T22:17:56.078 INFO:teuthology.orchestra.run.smithi067.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-22T22:17:56.078 INFO:teuthology.orchestra.run.smithi067.stderr:Mon IP `172.21.15.67` is in CIDR network `172.21.0.0/20` 2024-04-22T22:17:56.079 INFO:teuthology.orchestra.run.smithi067.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-04-22T22:17:56.079 INFO:teuthology.orchestra.run.smithi067.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-22T22:17:56.188 INFO:teuthology.orchestra.run.smithi067.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-04-22T22:17:56.916 INFO:teuthology.orchestra.run.smithi067.stderr:/bin/podman: Getting image source signatures 2024-04-22T22:17:56.916 INFO:teuthology.orchestra.run.smithi067.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-04-22T22:17:56.916 INFO:teuthology.orchestra.run.smithi067.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-04-22T22:17:56.916 INFO:teuthology.orchestra.run.smithi067.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-04-22T22:17:56.916 INFO:teuthology.orchestra.run.smithi067.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-04-22T22:17:56.916 INFO:teuthology.orchestra.run.smithi067.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-04-22T22:17:56.916 INFO:teuthology.orchestra.run.smithi067.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-04-22T22:17:57.101 INFO:teuthology.orchestra.run.smithi067.stderr:/bin/podman: Writing manifest to image destination 2024-04-22T22:17:57.246 INFO:teuthology.orchestra.run.smithi067.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-04-22T22:17:57.642 INFO:teuthology.orchestra.run.smithi067.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-04-22T22:17:58.153 INFO:teuthology.orchestra.run.smithi067.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-04-22T22:17:58.153 INFO:teuthology.orchestra.run.smithi067.stderr:Extracting ceph user uid/gid from container image... 2024-04-22T22:17:58.570 INFO:teuthology.orchestra.run.smithi067.stderr:stat: 167 167 2024-04-22T22:17:59.008 INFO:teuthology.orchestra.run.smithi067.stderr:Creating initial keys... 2024-04-22T22:17:59.474 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph-authtool: AQAX4iZm8R9MGRAAhvZzLuBMBjChdOyB1EcRNA== 2024-04-22T22:18:00.430 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph-authtool: AQAY4iZmSugKFhAAni5Lz21y8TRxFmn8dA/tDw== 2024-04-22T22:18:01.360 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph-authtool: AQAZ4iZm1Cr3EhAAXMdw0VoCzLPI8ejnMrMmgg== 2024-04-22T22:18:04.692 INFO:teuthology.orchestra.run.smithi067.stderr:Creating initial monmap... 2024-04-22T22:18:06.509 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-22T22:18:06.549 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-04-22T22:18:06.549 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 1ab1a3a8-00f6-11ef-bc93-c7b262605968 2024-04-22T22:18:06.549 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-22T22:18:06.965 INFO:teuthology.orchestra.run.smithi067.stderr:monmaptool for a [v2:172.21.15.67:3300,v1:172.21.15.67:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-22T22:18:06.965 INFO:teuthology.orchestra.run.smithi067.stderr:setting min_mon_release = octopus 2024-04-22T22:18:06.965 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/monmaptool: set fsid to 1ab1a3a8-00f6-11ef-bc93-c7b262605968 2024-04-22T22:18:06.965 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-22T22:18:06.966 INFO:teuthology.orchestra.run.smithi067.stderr: 2024-04-22T22:18:06.966 INFO:teuthology.orchestra.run.smithi067.stderr:Creating mon... 2024-04-22T22:18:07.889 INFO:teuthology.orchestra.run.smithi067.stderr:create mon.a on 2024-04-22T22:18:08.099 INFO:teuthology.orchestra.run.smithi067.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-04-22T22:18:08.327 INFO:teuthology.orchestra.run.smithi067.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968.target → /etc/systemd/system/ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968.target. 2024-04-22T22:18:08.327 INFO:teuthology.orchestra.run.smithi067.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968.target → /etc/systemd/system/ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968.target. 2024-04-22T22:18:08.629 INFO:teuthology.orchestra.run.smithi067.stderr:systemctl: Failed to reset failed state of unit ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@mon.a.service: Unit ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@mon.a.service not loaded. 2024-04-22T22:18:08.637 INFO:teuthology.orchestra.run.smithi067.stderr:systemctl: Created symlink /etc/systemd/system/ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968.target.wants/ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@mon.a.service → /etc/systemd/system/ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@.service. 2024-04-22T22:18:09.348 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:09 smithi067 ceph-mon[21515]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-22T22:18:09.348 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:09 smithi067 bash[21496]: 78aa3109e9233ca0165d88665f043f0f75c4592ea368d56c2b7d714c752a0d6d 2024-04-22T22:18:09.362 INFO:teuthology.orchestra.run.smithi067.stderr:systemctl: disabled 2024-04-22T22:18:09.367 INFO:teuthology.orchestra.run.smithi067.stderr:systemctl: inactive 2024-04-22T22:18:09.368 INFO:teuthology.orchestra.run.smithi067.stderr:firewalld.service is not enabled 2024-04-22T22:18:09.368 INFO:teuthology.orchestra.run.smithi067.stderr:Not possible to enable service . firewalld.service is not available 2024-04-22T22:18:09.368 INFO:teuthology.orchestra.run.smithi067.stderr:Waiting for mon to start... 2024-04-22T22:18:09.369 INFO:teuthology.orchestra.run.smithi067.stderr:Waiting for mon... 2024-04-22T22:18:09.626 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:09 smithi067 systemd[1]: Started Ceph mon.a for 1ab1a3a8-00f6-11ef-bc93-c7b262605968. 2024-04-22T22:18:09.805 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: cluster: 2024-04-22T22:18:09.805 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: id: 1ab1a3a8-00f6-11ef-bc93-c7b262605968 2024-04-22T22:18:09.805 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-04-22T22:18:09.805 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: 2024-04-22T22:18:09.805 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: services: 2024-04-22T22:18:09.805 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: mon: 1 daemons, quorum a (age 0.531498s) 2024-04-22T22:18:09.805 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: mgr: no daemons active 2024-04-22T22:18:09.805 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-04-22T22:18:09.806 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: 2024-04-22T22:18:09.806 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: data: 2024-04-22T22:18:09.806 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-04-22T22:18:09.806 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-04-22T22:18:09.806 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-04-22T22:18:09.806 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: pgs: 2024-04-22T22:18:09.806 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: 2024-04-22T22:18:10.679 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:10 smithi067 ceph-mon[21515]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-22T22:18:10.679 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:10 smithi067 ceph-mon[21515]: monmap e1: 1 mons at {a=[v2:172.21.15.67:3300/0,v1:172.21.15.67:6789/0]} 2024-04-22T22:18:10.680 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:10 smithi067 ceph-mon[21515]: fsmap 2024-04-22T22:18:10.680 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:10 smithi067 ceph-mon[21515]: osdmap e1: 0 total, 0 up, 0 in 2024-04-22T22:18:10.680 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:10 smithi067 ceph-mon[21515]: mgrmap e1: no daemons active 2024-04-22T22:18:10.680 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:10 smithi067 ceph-mon[21515]: from='client.? 172.21.15.67:0/2859341777' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-22T22:18:10.918 INFO:teuthology.orchestra.run.smithi067.stderr:mon is available 2024-04-22T22:18:10.918 INFO:teuthology.orchestra.run.smithi067.stderr:Assimilating anything we can from ceph.conf... 2024-04-22T22:18:11.399 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: 2024-04-22T22:18:11.399 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: [global] 2024-04-22T22:18:11.399 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: fsid = 1ab1a3a8-00f6-11ef-bc93-c7b262605968 2024-04-22T22:18:11.399 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.67:3300,v1:172.21.15.67:6789] 2024-04-22T22:18:11.399 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-04-22T22:18:11.399 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-04-22T22:18:11.399 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-04-22T22:18:11.400 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-04-22T22:18:11.400 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: 2024-04-22T22:18:11.400 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: [mgr] 2024-04-22T22:18:11.400 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: mgr/cephadm/use_agent = False 2024-04-22T22:18:11.400 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-04-22T22:18:11.400 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: 2024-04-22T22:18:11.400 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: [osd] 2024-04-22T22:18:11.400 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-04-22T22:18:11.400 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-04-22T22:18:11.542 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:11 smithi067 ceph-mon[21515]: from='client.? 172.21.15.67:0/1179300218' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-22T22:18:11.542 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:11 smithi067 ceph-mon[21515]: from='client.? 172.21.15.67:0/1179300218' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-22T22:18:12.190 INFO:teuthology.orchestra.run.smithi067.stderr:Generating new minimal ceph.conf... 2024-04-22T22:18:12.995 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:12 smithi067 ceph-mon[21515]: from='client.? 172.21.15.67:0/1318580222' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:18:13.230 INFO:teuthology.orchestra.run.smithi067.stderr:Restarting the monitor... 2024-04-22T22:18:13.414 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:13 smithi067 systemd[1]: Stopping Ceph mon.a for 1ab1a3a8-00f6-11ef-bc93-c7b262605968... 2024-04-22T22:18:13.414 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:13 smithi067 bash[21747]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mon.a" found: no such container 2024-04-22T22:18:13.705 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:13 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mon-a[21511]: 2024-04-22T22:18:13.436+0000 7fdac4d6a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T22:18:13.705 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:13 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mon-a[21511]: 2024-04-22T22:18:13.436+0000 7fdac4d6a700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-04-22T22:18:13.705 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:13 smithi067 podman[21758]: 2024-04-22 22:18:13.508476103 +0000 UTC m=+0.194084939 container died 78aa3109e9233ca0165d88665f043f0f75c4592ea368d56c2b7d714c752a0d6d (image=quay.io/ceph/ceph:v17.2.0, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mon-a, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , ceph=True, io.k8s.display-name=CentOS Stream 8, vcs-type=git, io.buildah.version=1.19.8, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, distribution-scope=public, name=centos-stream, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, io.openshift.expose-services=) 2024-04-22T22:18:13.995 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:13 smithi067 podman[21758]: 2024-04-22 22:18:13.811075038 +0000 UTC m=+0.496683865 container cleanup 78aa3109e9233ca0165d88665f043f0f75c4592ea368d56c2b7d714c752a0d6d (image=quay.io/ceph/ceph:v17.2.0, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mon-a, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, distribution-scope=public, name=centos-stream, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, GIT_BRANCH=HEAD, release=754, maintainer=Guillaume Abrioux , version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, com.redhat.component=centos-stream-container, ceph=True, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-22T22:18:13.995 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:13 smithi067 bash[21758]: ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mon-a 2024-04-22T22:18:13.995 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:13 smithi067 bash[21784]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mon.a" found: no such container 2024-04-22T22:18:14.345 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 podman[21772]: 2024-04-22 22:18:14.03684089 +0000 UTC m=+0.521988791 container remove 78aa3109e9233ca0165d88665f043f0f75c4592ea368d56c2b7d714c752a0d6d (image=quay.io/ceph/ceph:v17.2.0, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mon-a, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., name=centos-stream, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, distribution-scope=public, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, maintainer=Guillaume Abrioux , version=8, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-22T22:18:14.345 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 systemd[1]: ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@mon.a.service: Deactivated successfully. 2024-04-22T22:18:14.345 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 systemd[1]: Stopped Ceph mon.a for 1ab1a3a8-00f6-11ef-bc93-c7b262605968. 2024-04-22T22:18:14.345 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 systemd[1]: Starting Ceph mon.a for 1ab1a3a8-00f6-11ef-bc93-c7b262605968... 2024-04-22T22:18:14.345 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 podman[21856]: 2024-04-22 22:18:14.190427653 +0000 UTC m=+0.019636217 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-04-22T22:18:14.345 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 podman[21856]: 2024-04-22 22:18:14.287281621 +0000 UTC m=+0.116490187 container create fbf59dc0efb64c6a0df5907527f99215cee7b1d16f837b5b62e7cf0450d0b1df (image=quay.io/ceph/ceph:v17.2.0, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mon-a, GIT_CLEAN=True, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., release=754, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, maintainer=Guillaume Abrioux , name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-04-22T22:18:14.536 INFO:teuthology.orchestra.run.smithi067.stderr:Setting mon public_network to 172.21.0.0/20 2024-04-22T22:18:14.662 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 podman[21856]: 2024-04-22 22:18:14.46450426 +0000 UTC m=+0.293712823 container init fbf59dc0efb64c6a0df5907527f99215cee7b1d16f837b5b62e7cf0450d0b1df (image=quay.io/ceph/ceph:v17.2.0, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mon-a, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, RELEASE=HEAD, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, com.redhat.component=centos-stream-container, distribution-scope=public) 2024-04-22T22:18:14.662 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 podman[21856]: 2024-04-22 22:18:14.467599806 +0000 UTC m=+0.296808371 container start fbf59dc0efb64c6a0df5907527f99215cee7b1d16f837b5b62e7cf0450d0b1df (image=quay.io/ceph/ceph:v17.2.0, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mon-a, ceph=True, vendor=Red Hat, Inc., release=754, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, version=8, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, name=centos-stream, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8) 2024-04-22T22:18:14.662 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: set uid:gid to 167:167 (ceph:ceph) 2024-04-22T22:18:14.662 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-04-22T22:18:14.662 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: pidfile_write: ignore empty --pid-file 2024-04-22T22:18:14.662 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: load: jerasure load: lrc 2024-04-22T22:18:14.663 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: RocksDB version: 6.15.5 2024-04-22T22:18:14.663 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-22T22:18:14.663 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Compile date Apr 18 2022 2024-04-22T22:18:14.663 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: DB SUMMARY 2024-04-22T22:18:14.663 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: DB Session ID: 3PBENYLHVDHZGJ9LY4XP 2024-04-22T22:18:14.663 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: CURRENT file: CURRENT 2024-04-22T22:18:14.663 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: IDENTITY file: IDENTITY 2024-04-22T22:18:14.663 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-04-22T22:18:14.663 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-04-22T22:18:14.663 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 82635 ; 2024-04-22T22:18:14.663 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.error_if_exists: 0 2024-04-22T22:18:14.663 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.create_if_missing: 0 2024-04-22T22:18:14.663 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.paranoid_checks: 1 2024-04-22T22:18:14.663 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-22T22:18:14.663 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.env: 0x55e7a44c6860 2024-04-22T22:18:14.664 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.fs: Posix File System 2024-04-22T22:18:14.664 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.info_log: 0x55e7a6b13dc0 2024-04-22T22:18:14.664 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.max_file_opening_threads: 16 2024-04-22T22:18:14.664 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.statistics: (nil) 2024-04-22T22:18:14.664 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.use_fsync: 0 2024-04-22T22:18:14.664 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.max_log_file_size: 0 2024-04-22T22:18:14.664 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-22T22:18:14.664 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-22T22:18:14.664 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.keep_log_file_num: 1000 2024-04-22T22:18:14.664 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.recycle_log_file_num: 0 2024-04-22T22:18:14.664 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.allow_fallocate: 1 2024-04-22T22:18:14.664 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.allow_mmap_reads: 0 2024-04-22T22:18:14.664 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.allow_mmap_writes: 0 2024-04-22T22:18:14.664 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.use_direct_reads: 0 2024-04-22T22:18:14.664 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-22T22:18:14.664 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.create_missing_column_families: 0 2024-04-22T22:18:14.665 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.db_log_dir: 2024-04-22T22:18:14.665 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-04-22T22:18:14.665 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-22T22:18:14.665 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-22T22:18:14.665 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-22T22:18:14.665 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-22T22:18:14.665 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-22T22:18:14.665 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-22T22:18:14.665 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.advise_random_on_open: 1 2024-04-22T22:18:14.666 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.db_write_buffer_size: 0 2024-04-22T22:18:14.666 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.write_buffer_manager: 0x55e7a6c042a0 2024-04-22T22:18:14.666 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-22T22:18:14.666 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-22T22:18:14.667 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-22T22:18:14.667 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-22T22:18:14.667 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.rate_limiter: (nil) 2024-04-22T22:18:14.667 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-22T22:18:14.667 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.wal_recovery_mode: 2 2024-04-22T22:18:14.667 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.enable_thread_tracking: 0 2024-04-22T22:18:14.667 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.enable_pipelined_write: 0 2024-04-22T22:18:14.667 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.unordered_write: 0 2024-04-22T22:18:14.667 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-22T22:18:14.667 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-22T22:18:14.667 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-22T22:18:14.667 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-22T22:18:14.667 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.row_cache: None 2024-04-22T22:18:14.667 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.wal_filter: None 2024-04-22T22:18:14.667 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-22T22:18:14.668 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.allow_ingest_behind: 0 2024-04-22T22:18:14.668 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.preserve_deletes: 0 2024-04-22T22:18:14.668 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.two_write_queues: 0 2024-04-22T22:18:14.668 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.manual_wal_flush: 0 2024-04-22T22:18:14.668 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.atomic_flush: 0 2024-04-22T22:18:14.668 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-22T22:18:14.668 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-22T22:18:14.668 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-22T22:18:14.668 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.log_readahead_size: 0 2024-04-22T22:18:14.668 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-22T22:18:14.668 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.best_efforts_recovery: 0 2024-04-22T22:18:14.668 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-22T22:18:14.668 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-22T22:18:14.668 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.allow_data_in_errors: 0 2024-04-22T22:18:14.668 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.db_host_id: __hostname__ 2024-04-22T22:18:14.669 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.max_background_jobs: 2 2024-04-22T22:18:14.669 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.max_background_compactions: -1 2024-04-22T22:18:14.669 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.max_subcompactions: 1 2024-04-22T22:18:14.669 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-22T22:18:14.669 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-22T22:18:14.669 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-22T22:18:14.669 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.max_total_wal_size: 0 2024-04-22T22:18:14.669 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-22T22:18:14.669 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-22T22:18:14.669 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-22T22:18:14.669 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-22T22:18:14.669 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.max_open_files: -1 2024-04-22T22:18:14.669 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.bytes_per_sync: 0 2024-04-22T22:18:14.669 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-22T22:18:14.669 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-22T22:18:14.669 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.compaction_readahead_size: 0 2024-04-22T22:18:14.670 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.max_background_flushes: -1 2024-04-22T22:18:14.670 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Compression algorithms supported: 2024-04-22T22:18:14.670 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-22T22:18:14.670 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: kZSTD supported: 0 2024-04-22T22:18:14.670 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: kXpressCompression supported: 0 2024-04-22T22:18:14.670 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: kLZ4HCCompression supported: 1 2024-04-22T22:18:14.670 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: kLZ4Compression supported: 1 2024-04-22T22:18:14.670 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: kBZip2Compression supported: 0 2024-04-22T22:18:14.670 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: kZlibCompression supported: 1 2024-04-22T22:18:14.670 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: kSnappyCompression supported: 1 2024-04-22T22:18:14.670 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-22T22:18:14.670 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-04-22T22:18:14.670 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-04-22T22:18:14.670 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-22T22:18:14.670 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.merge_operator: 2024-04-22T22:18:14.671 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.compaction_filter: None 2024-04-22T22:18:14.671 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.compaction_filter_factory: None 2024-04-22T22:18:14.671 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.sst_partitioner_factory: None 2024-04-22T22:18:14.671 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-22T22:18:14.671 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-22T22:18:14.671 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55e7a6adfd00) 2024-04-22T22:18:14.671 INFO:journalctl@ceph.mon.a.smithi067.stdout: cache_index_and_filter_blocks: 1 2024-04-22T22:18:14.671 INFO:journalctl@ceph.mon.a.smithi067.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-22T22:18:14.671 INFO:journalctl@ceph.mon.a.smithi067.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-22T22:18:14.671 INFO:journalctl@ceph.mon.a.smithi067.stdout: pin_top_level_index_and_filter: 1 2024-04-22T22:18:14.671 INFO:journalctl@ceph.mon.a.smithi067.stdout: index_type: 0 2024-04-22T22:18:14.671 INFO:journalctl@ceph.mon.a.smithi067.stdout: data_block_index_type: 0 2024-04-22T22:18:14.671 INFO:journalctl@ceph.mon.a.smithi067.stdout: index_shortening: 1 2024-04-22T22:18:14.671 INFO:journalctl@ceph.mon.a.smithi067.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-22T22:18:14.671 INFO:journalctl@ceph.mon.a.smithi067.stdout: hash_index_allow_collision: 1 2024-04-22T22:18:14.672 INFO:journalctl@ceph.mon.a.smithi067.stdout: checksum: 1 2024-04-22T22:18:14.672 INFO:journalctl@ceph.mon.a.smithi067.stdout: no_block_cache: 0 2024-04-22T22:18:14.672 INFO:journalctl@ceph.mon.a.smithi067.stdout: block_cache: 0x55e7a6b4a170 2024-04-22T22:18:14.672 INFO:journalctl@ceph.mon.a.smithi067.stdout: block_cache_name: BinnedLRUCache 2024-04-22T22:18:14.672 INFO:journalctl@ceph.mon.a.smithi067.stdout: block_cache_options: 2024-04-22T22:18:14.672 INFO:journalctl@ceph.mon.a.smithi067.stdout: capacity : 536870912 2024-04-22T22:18:14.672 INFO:journalctl@ceph.mon.a.smithi067.stdout: num_shard_bits : 4 2024-04-22T22:18:14.672 INFO:journalctl@ceph.mon.a.smithi067.stdout: strict_capacity_limit : 0 2024-04-22T22:18:14.672 INFO:journalctl@ceph.mon.a.smithi067.stdout: high_pri_pool_ratio: 0.000 2024-04-22T22:18:14.672 INFO:journalctl@ceph.mon.a.smithi067.stdout: block_cache_compressed: (nil) 2024-04-22T22:18:14.672 INFO:journalctl@ceph.mon.a.smithi067.stdout: persistent_cache: (nil) 2024-04-22T22:18:14.672 INFO:journalctl@ceph.mon.a.smithi067.stdout: block_size: 4096 2024-04-22T22:18:14.672 INFO:journalctl@ceph.mon.a.smithi067.stdout: block_size_deviation: 10 2024-04-22T22:18:14.673 INFO:journalctl@ceph.mon.a.smithi067.stdout: block_restart_interval: 16 2024-04-22T22:18:14.673 INFO:journalctl@ceph.mon.a.smithi067.stdout: index_block_restart_interval: 1 2024-04-22T22:18:14.673 INFO:journalctl@ceph.mon.a.smithi067.stdout: metadata_block_size: 4096 2024-04-22T22:18:14.673 INFO:journalctl@ceph.mon.a.smithi067.stdout: partition_filters: 0 2024-04-22T22:18:14.673 INFO:journalctl@ceph.mon.a.smithi067.stdout: use_delta_encoding: 1 2024-04-22T22:18:14.673 INFO:journalctl@ceph.mon.a.smithi067.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-22T22:18:14.673 INFO:journalctl@ceph.mon.a.smithi067.stdout: whole_key_filtering: 1 2024-04-22T22:18:14.673 INFO:journalctl@ceph.mon.a.smithi067.stdout: verify_compression: 0 2024-04-22T22:18:14.673 INFO:journalctl@ceph.mon.a.smithi067.stdout: read_amp_bytes_per_bit: 0 2024-04-22T22:18:14.674 INFO:journalctl@ceph.mon.a.smithi067.stdout: format_version: 4 2024-04-22T22:18:14.674 INFO:journalctl@ceph.mon.a.smithi067.stdout: enable_index_compression: 1 2024-04-22T22:18:14.674 INFO:journalctl@ceph.mon.a.smithi067.stdout: block_align: 0 2024-04-22T22:18:14.674 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.write_buffer_size: 33554432 2024-04-22T22:18:14.674 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.max_write_buffer_number: 2 2024-04-22T22:18:14.674 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.compression: NoCompression 2024-04-22T22:18:14.674 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.bottommost_compression: Disabled 2024-04-22T22:18:14.674 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.prefix_extractor: nullptr 2024-04-22T22:18:14.674 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-22T22:18:14.674 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.num_levels: 7 2024-04-22T22:18:14.674 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-22T22:18:14.675 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-22T22:18:14.675 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-22T22:18:14.675 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-22T22:18:14.675 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-22T22:18:14.675 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-22T22:18:14.675 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-22T22:18:14.675 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-22T22:18:14.675 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-22T22:18:14.675 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-22T22:18:14.675 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-22T22:18:14.675 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.compression_opts.level: 32767 2024-04-22T22:18:14.675 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.compression_opts.strategy: 0 2024-04-22T22:18:14.675 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-22T22:18:14.675 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-22T22:18:14.675 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-22T22:18:14.675 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.compression_opts.enabled: false 2024-04-22T22:18:14.676 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-22T22:18:14.676 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-22T22:18:14.676 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-22T22:18:14.676 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.target_file_size_base: 67108864 2024-04-22T22:18:14.676 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-22T22:18:14.676 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-22T22:18:14.676 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-22T22:18:14.676 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-22T22:18:14.676 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-22T22:18:14.676 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-22T22:18:14.676 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-22T22:18:14.676 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-22T22:18:14.676 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-22T22:18:14.676 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-22T22:18:14.676 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-22T22:18:14.676 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-22T22:18:14.677 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-22T22:18:14.677 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.arena_block_size: 4194304 2024-04-22T22:18:14.677 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-22T22:18:14.677 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-22T22:18:14.677 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-22T22:18:14.677 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.disable_auto_compactions: 0 2024-04-22T22:18:14.677 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-22T22:18:14.677 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-22T22:18:14.677 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-22T22:18:14.677 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-22T22:18:14.677 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-22T22:18:14.677 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-22T22:18:14.677 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-22T22:18:14.677 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-22T22:18:14.677 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-22T22:18:14.677 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-22T22:18:14.678 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.table_properties_collectors: 2024-04-22T22:18:14.678 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.inplace_update_support: 0 2024-04-22T22:18:14.678 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-22T22:18:14.678 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-22T22:18:14.678 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-22T22:18:14.678 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-22T22:18:14.678 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.bloom_locality: 0 2024-04-22T22:18:14.678 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.max_successive_merges: 0 2024-04-22T22:18:14.678 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-22T22:18:14.678 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.paranoid_file_checks: 0 2024-04-22T22:18:14.678 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.force_consistency_checks: 1 2024-04-22T22:18:14.678 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.report_bg_io_stats: 0 2024-04-22T22:18:14.678 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.ttl: 2592000 2024-04-22T22:18:14.678 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-22T22:18:14.679 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.enable_blob_files: false 2024-04-22T22:18:14.679 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.min_blob_size: 0 2024-04-22T22:18:14.679 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.blob_file_size: 268435456 2024-04-22T22:18:14.679 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-22T22:18:14.679 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-22T22:18:14.679 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-22T22:18:14.679 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-22T22:18:14.679 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-04-22T22:18:14.679 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-04-22T22:18:14.679 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713824294493847, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-04-22T22:18:14.679 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-04-22T22:18:14.679 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-22T22:18:14.679 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713824294494586, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79185, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77448, "index_size": 225, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10027, "raw_average_key_size": 47, "raw_value_size": 71757, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 211, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1713824294, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "5a0aa890-eb67-46a6-9b41-9bb15673be3b", "db_session_id": "3PBENYLHVDHZGJ9LY4XP"}} 2024-04-22T22:18:14.679 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-04-22T22:18:14.679 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713824294494918, "job": 1, "event": "recovery_finished"} 2024-04-22T22:18:14.679 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-22T22:18:14.680 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55e7a6b30a80 2024-04-22T22:18:14.680 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: DB pointer 0x55e7a6b40000 2024-04-22T22:18:14.680 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-04-22T22:18:14.680 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-22T22:18:14.680 INFO:journalctl@ceph.mon.a.smithi067.stdout: ** DB Stats ** 2024-04-22T22:18:14.680 INFO:journalctl@ceph.mon.a.smithi067.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T22:18:14.680 INFO:journalctl@ceph.mon.a.smithi067.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-22T22:18:14.680 INFO:journalctl@ceph.mon.a.smithi067.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-22T22:18:14.680 INFO:journalctl@ceph.mon.a.smithi067.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T22:18:14.680 INFO:journalctl@ceph.mon.a.smithi067.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-22T22:18:14.680 INFO:journalctl@ceph.mon.a.smithi067.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-22T22:18:14.680 INFO:journalctl@ceph.mon.a.smithi067.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T22:18:14.680 INFO:journalctl@ceph.mon.a.smithi067.stdout: 2024-04-22T22:18:14.680 INFO:journalctl@ceph.mon.a.smithi067.stdout: ** Compaction Stats [default] ** 2024-04-22T22:18:14.680 INFO:journalctl@ceph.mon.a.smithi067.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T22:18:14.681 INFO:journalctl@ceph.mon.a.smithi067.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T22:18:14.681 INFO:journalctl@ceph.mon.a.smithi067.stdout: L0 2/0 79.09 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 158.3 0.00 0.00 1 0.000 0 0 2024-04-22T22:18:14.681 INFO:journalctl@ceph.mon.a.smithi067.stdout: Sum 2/0 79.09 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 158.3 0.00 0.00 1 0.000 0 0 2024-04-22T22:18:14.681 INFO:journalctl@ceph.mon.a.smithi067.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 158.3 0.00 0.00 1 0.000 0 0 2024-04-22T22:18:14.681 INFO:journalctl@ceph.mon.a.smithi067.stdout: 2024-04-22T22:18:14.681 INFO:journalctl@ceph.mon.a.smithi067.stdout: ** Compaction Stats [default] ** 2024-04-22T22:18:14.681 INFO:journalctl@ceph.mon.a.smithi067.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T22:18:14.681 INFO:journalctl@ceph.mon.a.smithi067.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T22:18:14.681 INFO:journalctl@ceph.mon.a.smithi067.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 158.3 0.00 0.00 1 0.000 0 0 2024-04-22T22:18:14.681 INFO:journalctl@ceph.mon.a.smithi067.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T22:18:14.681 INFO:journalctl@ceph.mon.a.smithi067.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-22T22:18:14.681 INFO:journalctl@ceph.mon.a.smithi067.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-22T22:18:14.681 INFO:journalctl@ceph.mon.a.smithi067.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-22T22:18:14.681 INFO:journalctl@ceph.mon.a.smithi067.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-22T22:18:14.681 INFO:journalctl@ceph.mon.a.smithi067.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-22T22:18:14.681 INFO:journalctl@ceph.mon.a.smithi067.stdout: Cumulative compaction: 0.00 GB write, 24.92 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T22:18:14.682 INFO:journalctl@ceph.mon.a.smithi067.stdout: Interval compaction: 0.00 GB write, 24.92 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T22:18:14.682 INFO:journalctl@ceph.mon.a.smithi067.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-22T22:18:14.682 INFO:journalctl@ceph.mon.a.smithi067.stdout: 2024-04-22T22:18:14.682 INFO:journalctl@ceph.mon.a.smithi067.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-22T22:18:14.682 INFO:journalctl@ceph.mon.a.smithi067.stdout: 2024-04-22T22:18:14.682 INFO:journalctl@ceph.mon.a.smithi067.stdout: ** Compaction Stats [default] ** 2024-04-22T22:18:14.682 INFO:journalctl@ceph.mon.a.smithi067.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T22:18:14.682 INFO:journalctl@ceph.mon.a.smithi067.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T22:18:14.682 INFO:journalctl@ceph.mon.a.smithi067.stdout: L0 2/0 79.09 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 158.3 0.00 0.00 1 0.000 0 0 2024-04-22T22:18:14.682 INFO:journalctl@ceph.mon.a.smithi067.stdout: Sum 2/0 79.09 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 158.3 0.00 0.00 1 0.000 0 0 2024-04-22T22:18:14.682 INFO:journalctl@ceph.mon.a.smithi067.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-22T22:18:14.683 INFO:journalctl@ceph.mon.a.smithi067.stdout: 2024-04-22T22:18:14.683 INFO:journalctl@ceph.mon.a.smithi067.stdout: ** Compaction Stats [default] ** 2024-04-22T22:18:14.683 INFO:journalctl@ceph.mon.a.smithi067.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T22:18:14.683 INFO:journalctl@ceph.mon.a.smithi067.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T22:18:14.683 INFO:journalctl@ceph.mon.a.smithi067.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 158.3 0.00 0.00 1 0.000 0 0 2024-04-22T22:18:14.683 INFO:journalctl@ceph.mon.a.smithi067.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T22:18:14.683 INFO:journalctl@ceph.mon.a.smithi067.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-22T22:18:14.683 INFO:journalctl@ceph.mon.a.smithi067.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-22T22:18:14.683 INFO:journalctl@ceph.mon.a.smithi067.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-22T22:18:14.683 INFO:journalctl@ceph.mon.a.smithi067.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-22T22:18:14.683 INFO:journalctl@ceph.mon.a.smithi067.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-22T22:18:14.683 INFO:journalctl@ceph.mon.a.smithi067.stdout: Cumulative compaction: 0.00 GB write, 24.67 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T22:18:14.683 INFO:journalctl@ceph.mon.a.smithi067.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T22:18:14.684 INFO:journalctl@ceph.mon.a.smithi067.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-22T22:18:14.684 INFO:journalctl@ceph.mon.a.smithi067.stdout: 2024-04-22T22:18:14.684 INFO:journalctl@ceph.mon.a.smithi067.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-22T22:18:14.684 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: starting mon.a rank 0 at public addrs [v2:172.21.15.67:3300/0,v1:172.21.15.67:6789/0] at bind addrs [v2:172.21.15.67:3300/0,v1:172.21.15.67:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 2024-04-22T22:18:14.684 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: mon.a@-1(???) e1 preinit fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 2024-04-22T22:18:14.684 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: mon.a@-1(???).mds e1 new map 2024-04-22T22:18:14.684 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: mon.a@-1(???).mds e1 print_map 2024-04-22T22:18:14.684 INFO:journalctl@ceph.mon.a.smithi067.stdout: e1 2024-04-22T22:18:14.684 INFO:journalctl@ceph.mon.a.smithi067.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-22T22:18:14.684 INFO:journalctl@ceph.mon.a.smithi067.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-22T22:18:14.684 INFO:journalctl@ceph.mon.a.smithi067.stdout: legacy client fscid: -1 2024-04-22T22:18:14.684 INFO:journalctl@ceph.mon.a.smithi067.stdout: 2024-04-22T22:18:14.684 INFO:journalctl@ceph.mon.a.smithi067.stdout: No filesystems configured 2024-04-22T22:18:14.684 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-22T22:18:14.685 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-22T22:18:14.685 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-22T22:18:14.685 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-22T22:18:14.685 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-04-22T22:18:14.685 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 bash[21856]: fbf59dc0efb64c6a0df5907527f99215cee7b1d16f837b5b62e7cf0450d0b1df 2024-04-22T22:18:14.685 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-22T22:18:14.685 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: monmap e1: 1 mons at {a=[v2:172.21.15.67:3300/0,v1:172.21.15.67:6789/0]} 2024-04-22T22:18:14.685 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: fsmap 2024-04-22T22:18:14.685 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: osdmap e1: 0 total, 0 up, 0 in 2024-04-22T22:18:14.685 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 ceph-mon[21875]: mgrmap e1: no daemons active 2024-04-22T22:18:14.685 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:14 smithi067 systemd[1]: Started Ceph mon.a for 1ab1a3a8-00f6-11ef-bc93-c7b262605968. 2024-04-22T22:18:15.728 INFO:teuthology.orchestra.run.smithi067.stderr:Wrote config to /etc/ceph/ceph.conf 2024-04-22T22:18:15.728 INFO:teuthology.orchestra.run.smithi067.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-04-22T22:18:15.728 INFO:teuthology.orchestra.run.smithi067.stderr:Creating mgr... 2024-04-22T22:18:15.729 INFO:teuthology.orchestra.run.smithi067.stderr:Verifying port 9283 ... 2024-04-22T22:18:15.914 INFO:teuthology.orchestra.run.smithi067.stderr:systemctl: Failed to reset failed state of unit ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@mgr.y.service: Unit ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@mgr.y.service not loaded. 2024-04-22T22:18:15.921 INFO:teuthology.orchestra.run.smithi067.stderr:systemctl: Created symlink /etc/systemd/system/ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968.target.wants/ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@mgr.y.service → /etc/systemd/system/ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@.service. 2024-04-22T22:18:16.245 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:16 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/3888440981' entity='client.admin' 2024-04-22T22:18:16.245 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:16 smithi067 systemd[1]: Starting Ceph mgr.y for 1ab1a3a8-00f6-11ef-bc93-c7b262605968... 2024-04-22T22:18:16.595 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:16 smithi067 podman[22087]: 2024-04-22 22:18:16.245222561 +0000 UTC m=+0.019168424 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-04-22T22:18:16.595 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:16 smithi067 podman[22087]: 2024-04-22 22:18:16.349078881 +0000 UTC m=+0.123024717 container create c5a816fd6ddeadf20fc00707d927f83d3e307f72801ab39536be5b0e0ec82140 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, vcs-type=git, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, architecture=x86_64, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, GIT_CLEAN=True, version=8, ceph=True, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container) 2024-04-22T22:18:16.595 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:16 smithi067 podman[22087]: 2024-04-22 22:18:16.517855674 +0000 UTC m=+0.291801513 container init c5a816fd6ddeadf20fc00707d927f83d3e307f72801ab39536be5b0e0ec82140 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, GIT_CLEAN=True, version=8, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, distribution-scope=public, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-22T22:18:16.595 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:16 smithi067 podman[22087]: 2024-04-22 22:18:16.521122124 +0000 UTC m=+0.295067961 container start c5a816fd6ddeadf20fc00707d927f83d3e307f72801ab39536be5b0e0ec82140 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y, com.redhat.component=centos-stream-container, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, name=centos-stream, architecture=x86_64, version=8, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, io.buildah.version=1.19.8, GIT_CLEAN=True, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0) 2024-04-22T22:18:16.595 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:16 smithi067 bash[22087]: c5a816fd6ddeadf20fc00707d927f83d3e307f72801ab39536be5b0e0ec82140 2024-04-22T22:18:16.607 INFO:teuthology.orchestra.run.smithi067.stderr:systemctl: disabled 2024-04-22T22:18:16.614 INFO:teuthology.orchestra.run.smithi067.stderr:systemctl: inactive 2024-04-22T22:18:16.615 INFO:teuthology.orchestra.run.smithi067.stderr:firewalld.service is not enabled 2024-04-22T22:18:16.615 INFO:teuthology.orchestra.run.smithi067.stderr:Not possible to enable service . firewalld.service is not available 2024-04-22T22:18:16.623 INFO:teuthology.orchestra.run.smithi067.stderr:systemctl: disabled 2024-04-22T22:18:16.631 INFO:teuthology.orchestra.run.smithi067.stderr:systemctl: inactive 2024-04-22T22:18:16.632 INFO:teuthology.orchestra.run.smithi067.stderr:firewalld.service is not enabled 2024-04-22T22:18:16.632 INFO:teuthology.orchestra.run.smithi067.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-04-22T22:18:16.632 INFO:teuthology.orchestra.run.smithi067.stderr:Waiting for mgr to start... 2024-04-22T22:18:16.632 INFO:teuthology.orchestra.run.smithi067.stderr:Waiting for mgr... 2024-04-22T22:18:16.880 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:16 smithi067 systemd[1]: Started Ceph mgr.y for 1ab1a3a8-00f6-11ef-bc93-c7b262605968. 2024-04-22T22:18:16.880 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:16 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:16.668+0000 7f48e5f2a000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-22T22:18:16.880 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:16 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:16.809+0000 7f48e5f2a000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-22T22:18:17.168 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:16 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:16.912+0000 7f48e5f2a000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-22T22:18:17.174 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: 2024-04-22T22:18:17.175 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: { 2024-04-22T22:18:17.175 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "fsid": "1ab1a3a8-00f6-11ef-bc93-c7b262605968", 2024-04-22T22:18:17.175 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "health": { 2024-04-22T22:18:17.175 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-22T22:18:17.175 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "checks": {}, 2024-04-22T22:18:17.175 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "mutes": [] 2024-04-22T22:18:17.175 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: }, 2024-04-22T22:18:17.175 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-22T22:18:17.175 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "quorum": [ 2024-04-22T22:18:17.175 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: 0 2024-04-22T22:18:17.175 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: ], 2024-04-22T22:18:17.175 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-22T22:18:17.175 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "a" 2024-04-22T22:18:17.175 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: ], 2024-04-22T22:18:17.175 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-04-22T22:18:17.176 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "monmap": { 2024-04-22T22:18:17.176 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T22:18:17.176 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-22T22:18:17.176 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-22T22:18:17.176 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: }, 2024-04-22T22:18:17.176 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "osdmap": { 2024-04-22T22:18:17.176 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T22:18:17.176 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-22T22:18:17.176 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-22T22:18:17.176 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-22T22:18:17.176 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-22T22:18:17.176 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-22T22:18:17.176 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-22T22:18:17.176 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: }, 2024-04-22T22:18:17.176 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "pgmap": { 2024-04-22T22:18:17.176 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-22T22:18:17.177 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-22T22:18:17.177 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-22T22:18:17.177 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-22T22:18:17.177 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-22T22:18:17.177 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-22T22:18:17.177 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-22T22:18:17.177 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-22T22:18:17.177 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: }, 2024-04-22T22:18:17.177 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "fsmap": { 2024-04-22T22:18:17.177 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T22:18:17.177 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-22T22:18:17.177 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-22T22:18:17.177 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: }, 2024-04-22T22:18:17.177 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-22T22:18:17.178 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "available": false, 2024-04-22T22:18:17.178 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-22T22:18:17.178 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "modules": [ 2024-04-22T22:18:17.178 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "iostat", 2024-04-22T22:18:17.178 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "nfs", 2024-04-22T22:18:17.178 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "restful" 2024-04-22T22:18:17.178 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: ], 2024-04-22T22:18:17.178 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "services": {} 2024-04-22T22:18:17.178 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: }, 2024-04-22T22:18:17.178 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "servicemap": { 2024-04-22T22:18:17.178 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T22:18:17.178 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "modified": "2024-04-22T22:18:09.297404+0000", 2024-04-22T22:18:17.178 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "services": {} 2024-04-22T22:18:17.178 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: }, 2024-04-22T22:18:17.179 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-22T22:18:17.179 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: } 2024-04-22T22:18:17.494 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:17 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/2960652226' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-22T22:18:17.495 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:17 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:17.242+0000 7f48e5f2a000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-22T22:18:17.495 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:17 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:17.327+0000 7f48e5f2a000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-22T22:18:17.772 INFO:teuthology.orchestra.run.smithi067.stderr:mgr not available, waiting (1/15)... 2024-04-22T22:18:18.753 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:18 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:18.522+0000 7f48e5f2a000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-22T22:18:18.753 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:18 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:18.691+0000 7f48e5f2a000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-22T22:18:18.753 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:18 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:18.775+0000 7f48e5f2a000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-22T22:18:19.495 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:19 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:19.098+0000 7f48e5f2a000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-22T22:18:19.495 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:19 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:19.190+0000 7f48e5f2a000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-22T22:18:19.495 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:19 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:19.285+0000 7f48e5f2a000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-22T22:18:19.903 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:19 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:19.534+0000 7f48e5f2a000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-22T22:18:20.179 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:20 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:20.001+0000 7f48e5f2a000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-22T22:18:20.300 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: 2024-04-22T22:18:20.300 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: { 2024-04-22T22:18:20.300 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "fsid": "1ab1a3a8-00f6-11ef-bc93-c7b262605968", 2024-04-22T22:18:20.300 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "health": { 2024-04-22T22:18:20.300 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-22T22:18:20.300 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "checks": {}, 2024-04-22T22:18:20.300 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "mutes": [] 2024-04-22T22:18:20.300 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: }, 2024-04-22T22:18:20.300 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-22T22:18:20.300 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "quorum": [ 2024-04-22T22:18:20.300 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: 0 2024-04-22T22:18:20.301 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: ], 2024-04-22T22:18:20.301 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-22T22:18:20.301 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "a" 2024-04-22T22:18:20.301 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: ], 2024-04-22T22:18:20.301 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "quorum_age": 5, 2024-04-22T22:18:20.301 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "monmap": { 2024-04-22T22:18:20.301 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T22:18:20.301 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-22T22:18:20.301 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-22T22:18:20.301 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: }, 2024-04-22T22:18:20.301 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "osdmap": { 2024-04-22T22:18:20.301 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T22:18:20.301 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-22T22:18:20.301 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-22T22:18:20.301 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-22T22:18:20.302 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-22T22:18:20.302 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-22T22:18:20.302 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-22T22:18:20.302 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: }, 2024-04-22T22:18:20.302 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "pgmap": { 2024-04-22T22:18:20.302 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-22T22:18:20.302 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-22T22:18:20.302 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-22T22:18:20.302 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-22T22:18:20.302 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-22T22:18:20.302 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-22T22:18:20.302 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-22T22:18:20.302 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-22T22:18:20.302 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: }, 2024-04-22T22:18:20.303 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "fsmap": { 2024-04-22T22:18:20.303 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T22:18:20.303 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-22T22:18:20.303 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-22T22:18:20.303 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: }, 2024-04-22T22:18:20.303 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-22T22:18:20.303 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "available": false, 2024-04-22T22:18:20.303 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-22T22:18:20.303 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "modules": [ 2024-04-22T22:18:20.303 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "iostat", 2024-04-22T22:18:20.303 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "nfs", 2024-04-22T22:18:20.303 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "restful" 2024-04-22T22:18:20.303 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: ], 2024-04-22T22:18:20.303 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "services": {} 2024-04-22T22:18:20.303 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: }, 2024-04-22T22:18:20.303 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "servicemap": { 2024-04-22T22:18:20.304 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T22:18:20.304 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "modified": "2024-04-22T22:18:09.297404+0000", 2024-04-22T22:18:20.304 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "services": {} 2024-04-22T22:18:20.304 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: }, 2024-04-22T22:18:20.304 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-22T22:18:20.304 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: } 2024-04-22T22:18:20.745 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:20 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/731668112' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-22T22:18:20.745 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:20 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:20.517+0000 7f48e5f2a000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-22T22:18:20.745 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:20 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:20.614+0000 7f48e5f2a000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-22T22:18:21.026 INFO:teuthology.orchestra.run.smithi067.stderr:mgr not available, waiting (2/15)... 2024-04-22T22:18:21.245 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:20 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:20.919+0000 7f48e5f2a000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-22T22:18:21.245 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:21 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:21.011+0000 7f48e5f2a000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-22T22:18:21.745 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:21 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:21.428+0000 7f48e5f2a000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-22T22:18:21.745 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:21 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:21.565+0000 7f48e5f2a000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-22T22:18:21.745 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:21 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:21.659+0000 7f48e5f2a000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-22T22:18:22.245 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:21 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:21.776+0000 7f48e5f2a000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-22T22:18:22.245 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:21 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:21.867+0000 7f48e5f2a000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-22T22:18:22.745 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:22 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:22.435+0000 7f48e5f2a000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-22T22:18:23.153 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:22 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:22.807+0000 7f48e5f2a000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-22T22:18:23.153 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:22 smithi067 ceph-mon[21875]: Activating manager daemon y 2024-04-22T22:18:23.153 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:22 smithi067 ceph-mon[21875]: mgrmap e2: y(active, starting, since 0.00262747s) 2024-04-22T22:18:23.153 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:22 smithi067 ceph-mon[21875]: from='mgr.14100 172.21.15.67:0/1723388810' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T22:18:23.153 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:22 smithi067 ceph-mon[21875]: from='mgr.14100 172.21.15.67:0/1723388810' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T22:18:23.153 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:22 smithi067 ceph-mon[21875]: from='mgr.14100 172.21.15.67:0/1723388810' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T22:18:23.153 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:22 smithi067 ceph-mon[21875]: from='mgr.14100 172.21.15.67:0/1723388810' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T22:18:23.153 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:22 smithi067 ceph-mon[21875]: from='mgr.14100 172.21.15.67:0/1723388810' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T22:18:23.153 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:22 smithi067 ceph-mon[21875]: Manager daemon y is now available 2024-04-22T22:18:23.154 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:22 smithi067 ceph-mon[21875]: from='mgr.14100 172.21.15.67:0/1723388810' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:18:23.154 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:22 smithi067 ceph-mon[21875]: from='mgr.14100 172.21.15.67:0/1723388810' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:18:23.154 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:22 smithi067 ceph-mon[21875]: from='mgr.14100 172.21.15.67:0/1723388810' entity='mgr.y' 2024-04-22T22:18:23.154 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:22 smithi067 ceph-mon[21875]: from='mgr.14100 172.21.15.67:0/1723388810' entity='mgr.y' 2024-04-22T22:18:23.154 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:22 smithi067 ceph-mon[21875]: from='mgr.14100 172.21.15.67:0/1723388810' entity='mgr.y' 2024-04-22T22:18:23.555 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: 2024-04-22T22:18:23.555 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: { 2024-04-22T22:18:23.555 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "fsid": "1ab1a3a8-00f6-11ef-bc93-c7b262605968", 2024-04-22T22:18:23.555 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "health": { 2024-04-22T22:18:23.555 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-22T22:18:23.555 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "checks": {}, 2024-04-22T22:18:23.555 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "mutes": [] 2024-04-22T22:18:23.555 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: }, 2024-04-22T22:18:23.555 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-22T22:18:23.555 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "quorum": [ 2024-04-22T22:18:23.555 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: 0 2024-04-22T22:18:23.556 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: ], 2024-04-22T22:18:23.556 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-22T22:18:23.556 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "a" 2024-04-22T22:18:23.556 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: ], 2024-04-22T22:18:23.556 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "quorum_age": 9, 2024-04-22T22:18:23.556 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "monmap": { 2024-04-22T22:18:23.556 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T22:18:23.556 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-22T22:18:23.556 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-22T22:18:23.556 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: }, 2024-04-22T22:18:23.556 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "osdmap": { 2024-04-22T22:18:23.556 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T22:18:23.556 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-22T22:18:23.556 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-22T22:18:23.557 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-22T22:18:23.557 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-22T22:18:23.561 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-22T22:18:23.561 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-22T22:18:23.561 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: }, 2024-04-22T22:18:23.561 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "pgmap": { 2024-04-22T22:18:23.561 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-22T22:18:23.561 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-22T22:18:23.562 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-22T22:18:23.562 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-22T22:18:23.562 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-22T22:18:23.562 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-22T22:18:23.562 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-22T22:18:23.562 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-22T22:18:23.562 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: }, 2024-04-22T22:18:23.562 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "fsmap": { 2024-04-22T22:18:23.562 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T22:18:23.562 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-22T22:18:23.562 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-22T22:18:23.563 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: }, 2024-04-22T22:18:23.563 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-22T22:18:23.563 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "available": false, 2024-04-22T22:18:23.563 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-22T22:18:23.563 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "modules": [ 2024-04-22T22:18:23.563 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "iostat", 2024-04-22T22:18:23.563 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "nfs", 2024-04-22T22:18:23.563 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "restful" 2024-04-22T22:18:23.563 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: ], 2024-04-22T22:18:23.563 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "services": {} 2024-04-22T22:18:23.563 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: }, 2024-04-22T22:18:23.563 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "servicemap": { 2024-04-22T22:18:23.563 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T22:18:23.563 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "modified": "2024-04-22T22:18:09.297404+0000", 2024-04-22T22:18:23.563 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "services": {} 2024-04-22T22:18:23.563 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: }, 2024-04-22T22:18:23.564 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-22T22:18:23.564 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: } 2024-04-22T22:18:24.083 INFO:teuthology.orchestra.run.smithi067.stderr:mgr not available, waiting (3/15)... 2024-04-22T22:18:24.177 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:23 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/97317799' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-22T22:18:24.178 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:23 smithi067 ceph-mon[21875]: mgrmap e3: y(active, since 1.00586s) 2024-04-22T22:18:26.745 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:26 smithi067 ceph-mon[21875]: mgrmap e4: y(active, since 2s) 2024-04-22T22:18:26.758 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: 2024-04-22T22:18:26.758 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: { 2024-04-22T22:18:26.758 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "fsid": "1ab1a3a8-00f6-11ef-bc93-c7b262605968", 2024-04-22T22:18:26.758 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "health": { 2024-04-22T22:18:26.758 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-22T22:18:26.758 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "checks": {}, 2024-04-22T22:18:26.758 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "mutes": [] 2024-04-22T22:18:26.758 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: }, 2024-04-22T22:18:26.758 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-22T22:18:26.759 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "quorum": [ 2024-04-22T22:18:26.759 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: 0 2024-04-22T22:18:26.759 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: ], 2024-04-22T22:18:26.759 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-22T22:18:26.759 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "a" 2024-04-22T22:18:26.759 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: ], 2024-04-22T22:18:26.759 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "quorum_age": 12, 2024-04-22T22:18:26.759 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "monmap": { 2024-04-22T22:18:26.759 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T22:18:26.759 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-22T22:18:26.759 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-22T22:18:26.759 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: }, 2024-04-22T22:18:26.759 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "osdmap": { 2024-04-22T22:18:26.759 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T22:18:26.759 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-22T22:18:26.759 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-22T22:18:26.759 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-22T22:18:26.760 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-22T22:18:26.764 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-22T22:18:26.764 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-22T22:18:26.764 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: }, 2024-04-22T22:18:26.764 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "pgmap": { 2024-04-22T22:18:26.764 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-22T22:18:26.765 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-22T22:18:26.765 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-22T22:18:26.765 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-22T22:18:26.765 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-22T22:18:26.765 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-22T22:18:26.765 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-22T22:18:26.765 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-22T22:18:26.765 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: }, 2024-04-22T22:18:26.765 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "fsmap": { 2024-04-22T22:18:26.765 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T22:18:26.766 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-22T22:18:26.766 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-22T22:18:26.766 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: }, 2024-04-22T22:18:26.766 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-22T22:18:26.766 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "available": true, 2024-04-22T22:18:26.766 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-22T22:18:26.766 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "modules": [ 2024-04-22T22:18:26.766 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "iostat", 2024-04-22T22:18:26.766 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "nfs", 2024-04-22T22:18:26.767 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "restful" 2024-04-22T22:18:26.767 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: ], 2024-04-22T22:18:26.767 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "services": {} 2024-04-22T22:18:26.767 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: }, 2024-04-22T22:18:26.767 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "servicemap": { 2024-04-22T22:18:26.767 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-22T22:18:26.767 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "modified": "2024-04-22T22:18:09.297404+0000", 2024-04-22T22:18:26.767 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "services": {} 2024-04-22T22:18:26.767 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: }, 2024-04-22T22:18:26.768 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-22T22:18:26.768 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: } 2024-04-22T22:18:27.356 INFO:teuthology.orchestra.run.smithi067.stderr:mgr is available 2024-04-22T22:18:27.673 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:27 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/4293763383' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-22T22:18:27.966 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: 2024-04-22T22:18:27.966 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: [global] 2024-04-22T22:18:27.967 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: fsid = 1ab1a3a8-00f6-11ef-bc93-c7b262605968 2024-04-22T22:18:27.967 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-04-22T22:18:27.967 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-04-22T22:18:27.967 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-04-22T22:18:27.967 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-04-22T22:18:27.967 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: 2024-04-22T22:18:27.968 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: [mgr] 2024-04-22T22:18:27.968 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-04-22T22:18:27.968 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: 2024-04-22T22:18:27.968 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: [osd] 2024-04-22T22:18:27.968 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-04-22T22:18:27.968 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-04-22T22:18:28.529 INFO:teuthology.orchestra.run.smithi067.stderr:Enabling cephadm module... 2024-04-22T22:18:28.734 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:28 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/3966206074' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-22T22:18:28.734 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:28 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/3966206074' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-22T22:18:29.745 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:29 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/3575851242' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-04-22T22:18:29.745 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:29 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ignoring --setuser ceph since I am not root 2024-04-22T22:18:29.745 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:29 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ignoring --setgroup ceph since I am not root 2024-04-22T22:18:29.746 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:29 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:29.677+0000 7fe4dbfc9000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-22T22:18:30.091 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:29 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:29.811+0000 7fe4dbfc9000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-22T22:18:30.092 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:29 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:29.914+0000 7fe4dbfc9000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-22T22:18:30.365 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:30 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:30.251+0000 7fe4dbfc9000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-22T22:18:30.365 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:30 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:30.340+0000 7fe4dbfc9000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-22T22:18:30.745 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:30 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/3575851242' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-04-22T22:18:30.745 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:30 smithi067 ceph-mon[21875]: mgrmap e5: y(active, since 6s) 2024-04-22T22:18:30.853 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: { 2024-04-22T22:18:30.853 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "epoch": 5, 2024-04-22T22:18:30.853 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "available": true, 2024-04-22T22:18:30.853 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "active_name": "y", 2024-04-22T22:18:30.853 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-22T22:18:30.853 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: } 2024-04-22T22:18:31.434 INFO:teuthology.orchestra.run.smithi067.stderr:Waiting for the mgr to restart... 2024-04-22T22:18:31.434 INFO:teuthology.orchestra.run.smithi067.stderr:Waiting for mgr epoch 5... 2024-04-22T22:18:31.684 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:31 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/2885619468' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-22T22:18:31.685 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:31 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:31.542+0000 7fe4dbfc9000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-22T22:18:31.996 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:31 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:31.717+0000 7fe4dbfc9000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-22T22:18:31.996 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:31 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:31.802+0000 7fe4dbfc9000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-22T22:18:32.495 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:32 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:32.129+0000 7fe4dbfc9000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-22T22:18:32.495 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:32 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:32.223+0000 7fe4dbfc9000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-22T22:18:32.496 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:32 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:32.319+0000 7fe4dbfc9000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-22T22:18:32.995 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:32 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:32.568+0000 7fe4dbfc9000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-22T22:18:33.495 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:33 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:33.028+0000 7fe4dbfc9000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-22T22:18:33.913 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:33 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:33.530+0000 7fe4dbfc9000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-22T22:18:33.913 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:33 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:33.627+0000 7fe4dbfc9000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-22T22:18:33.913 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:33 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:33.934+0000 7fe4dbfc9000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-22T22:18:34.245 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:34 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:34.026+0000 7fe4dbfc9000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-22T22:18:34.745 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:34 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:34.441+0000 7fe4dbfc9000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-22T22:18:34.746 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:34 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:34.578+0000 7fe4dbfc9000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-22T22:18:34.746 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:34 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:34.674+0000 7fe4dbfc9000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-22T22:18:35.245 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:34 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:34.792+0000 7fe4dbfc9000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-22T22:18:35.246 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:34 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:34.883+0000 7fe4dbfc9000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-22T22:18:35.745 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:35.453+0000 7fe4dbfc9000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-22T22:18:36.245 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:35 smithi067 ceph-mon[21875]: Active manager daemon y restarted 2024-04-22T22:18:36.246 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:35 smithi067 ceph-mon[21875]: Activating manager daemon y 2024-04-22T22:18:36.246 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:35 smithi067 ceph-mon[21875]: osdmap e2: 0 total, 0 up, 0 in 2024-04-22T22:18:36.246 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:35.828+0000 7fe4dbfc9000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-22T22:18:36.876 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: { 2024-04-22T22:18:36.876 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-04-22T22:18:36.877 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "initialized": true 2024-04-22T22:18:36.877 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: } 2024-04-22T22:18:37.179 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:36 smithi067 ceph-mon[21875]: mgrmap e6: y(active, starting, since 0.0525135s) 2024-04-22T22:18:37.179 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:36 smithi067 ceph-mon[21875]: from='mgr.14120 172.21.15.67:0/3823221475' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T22:18:37.179 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:36 smithi067 ceph-mon[21875]: from='mgr.14120 172.21.15.67:0/3823221475' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T22:18:37.179 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:36 smithi067 ceph-mon[21875]: from='mgr.14120 172.21.15.67:0/3823221475' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T22:18:37.179 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:36 smithi067 ceph-mon[21875]: from='mgr.14120 172.21.15.67:0/3823221475' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T22:18:37.179 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:36 smithi067 ceph-mon[21875]: from='mgr.14120 172.21.15.67:0/3823221475' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T22:18:37.179 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:36 smithi067 ceph-mon[21875]: Manager daemon y is now available 2024-04-22T22:18:37.179 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:36 smithi067 ceph-mon[21875]: from='mgr.14120 172.21.15.67:0/3823221475' entity='mgr.y' 2024-04-22T22:18:37.179 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:36 smithi067 ceph-mon[21875]: from='mgr.14120 172.21.15.67:0/3823221475' entity='mgr.y' 2024-04-22T22:18:37.180 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:36 smithi067 ceph-mon[21875]: from='mgr.14120 172.21.15.67:0/3823221475' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:18:37.180 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:36 smithi067 ceph-mon[21875]: from='mgr.14120 172.21.15.67:0/3823221475' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:18:37.180 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:36 smithi067 ceph-mon[21875]: from='mgr.14120 172.21.15.67:0/3823221475' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:18:37.180 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:36 smithi067 ceph-mon[21875]: from='mgr.14120 172.21.15.67:0/3823221475' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:18:37.180 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:36 smithi067 ceph-mon[21875]: from='mgr.14120 172.21.15.67:0/3823221475' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:18:37.382 INFO:teuthology.orchestra.run.smithi067.stderr:mgr epoch 5 is available 2024-04-22T22:18:37.382 INFO:teuthology.orchestra.run.smithi067.stderr:Setting orchestrator backend to cephadm... 2024-04-22T22:18:37.495 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:37 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: [22/Apr/2024:22:18:37] ENGINE Bus STARTING 2024-04-22T22:18:37.824 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:37 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: [22/Apr/2024:22:18:37] ENGINE Serving on https://172.21.15.67:7150 2024-04-22T22:18:37.824 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:37 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: [22/Apr/2024:22:18:37] ENGINE Bus STARTED 2024-04-22T22:18:38.246 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:37 smithi067 ceph-mon[21875]: mgrmap e7: y(active, since 1.06114s) 2024-04-22T22:18:38.246 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:37 smithi067 ceph-mon[21875]: [22/Apr/2024:22:18:37] ENGINE Bus STARTING 2024-04-22T22:18:38.246 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:37 smithi067 ceph-mon[21875]: [22/Apr/2024:22:18:37] ENGINE Serving on https://172.21.15.67:7150 2024-04-22T22:18:38.246 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:37 smithi067 ceph-mon[21875]: [22/Apr/2024:22:18:37] ENGINE Bus STARTED 2024-04-22T22:18:38.246 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:37 smithi067 ceph-mon[21875]: from='mgr.14120 172.21.15.67:0/3823221475' entity='mgr.y' 2024-04-22T22:18:38.246 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:37 smithi067 ceph-mon[21875]: from='mgr.14120 172.21.15.67:0/3823221475' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:18:39.205 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:39 smithi067 ceph-mon[21875]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:18:39.205 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:39 smithi067 ceph-mon[21875]: from='mgr.14120 172.21.15.67:0/3823221475' entity='mgr.y' 2024-04-22T22:18:39.205 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:39 smithi067 ceph-mon[21875]: from='mgr.14120 172.21.15.67:0/3823221475' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:18:39.205 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:39 smithi067 ceph-mon[21875]: mgrmap e8: y(active, since 2s) 2024-04-22T22:18:39.205 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: value unchanged 2024-04-22T22:18:39.744 INFO:teuthology.orchestra.run.smithi067.stderr:Generating ssh key... 2024-04-22T22:18:40.746 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:40 smithi067 ceph-mon[21875]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:18:41.373 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:41 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: Generating public/private rsa key pair. 2024-04-22T22:18:41.373 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:41 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: Your identification has been saved in /tmp/tmp32ol9dc2/key. 2024-04-22T22:18:41.373 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:41 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: Your public key has been saved in /tmp/tmp32ol9dc2/key.pub. 2024-04-22T22:18:41.373 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:41 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: The key fingerprint is: 2024-04-22T22:18:41.373 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:41 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: SHA256:VNh3u8Qg7Z/4OOh7NI1xU3BU1ernI8x2zRDT05lXyLY ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968 2024-04-22T22:18:41.373 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:41 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: The key's randomart image is: 2024-04-22T22:18:41.373 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:41 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: +---[RSA 3072]----+ 2024-04-22T22:18:41.373 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:41 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: | o.. ..+O| 2024-04-22T22:18:41.373 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:41 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: | ..o + =.+| 2024-04-22T22:18:41.374 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:41 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: | . + = *=| 2024-04-22T22:18:41.374 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:41 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: | . o E++| 2024-04-22T22:18:41.374 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:41 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: | S X *o| 2024-04-22T22:18:41.374 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:41 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: | = B .| 2024-04-22T22:18:41.374 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:41 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: | oo+ =.| 2024-04-22T22:18:41.374 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:41 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: | . +=.o+| 2024-04-22T22:18:41.374 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:41 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: | .oo..o .| 2024-04-22T22:18:41.374 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:41 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: +----[SHA256]-----+ 2024-04-22T22:18:41.640 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:41 smithi067 ceph-mon[21875]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:18:41.640 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:41 smithi067 ceph-mon[21875]: Generating ssh key... 2024-04-22T22:18:41.640 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:41 smithi067 ceph-mon[21875]: from='mgr.14120 172.21.15.67:0/3823221475' entity='mgr.y' 2024-04-22T22:18:41.640 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:41 smithi067 ceph-mon[21875]: from='mgr.14120 172.21.15.67:0/3823221475' entity='mgr.y' 2024-04-22T22:18:42.278 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDTfcnI5qEd4T4D2qoXhOFIouxajTbw4+dH65fR/Qv49JPh05gUQnfNJg0JaWsrcmLTwiUKCKC0U2wLqtNeYEgvI+QyYMtMNO+yrEP4xJr1riBsV71WC4FIYG3+OkNEcNerOx3bW18JsaUmeM7ZfqCzOpe66me/+ukhH+Lsp4VGszKZ+/RSmTKt8mcadLdiqewg0C5WVsYZeE9FoNwbWxqoEKIB2oo8+72wGsOBRjoySFMc3ddtGSqj+JVUjOIjfwTNRGFHoQ71VTyYS3uq1pE8CuI8O+c63fUBpw7FECOMrHZf0w1tIuvVxjns3UKV4teF9/AD4Va1WGN2XqHN2hcgcAqHnRJ7Br/QhX0nfi4L9fVykV9ktRytrT1jBoB8+NF3Xqm4zptscfHd80K/bVSP+47VLmVj4in4yrk+k1aDTzXVdVoYsO5Db2RH11RlXbbKIaXzTSRpJqFsk41Ewp5HxGXtgOCoMcNdFTHJr7uqbCGMHJuQAhf3jIbtfZk+WKE= ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968 2024-04-22T22:18:42.846 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:42 smithi067 ceph-mon[21875]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:18:42.849 INFO:teuthology.orchestra.run.smithi067.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-04-22T22:18:42.850 INFO:teuthology.orchestra.run.smithi067.stderr:Adding key to root@localhost authorized_keys... 2024-04-22T22:18:42.850 INFO:teuthology.orchestra.run.smithi067.stderr:Adding host smithi067... 2024-04-22T22:18:43.714 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:43 smithi067 ceph-mon[21875]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi067", "addr": "172.21.15.67", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:18:44.311 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: Added host 'smithi067' with addr '172.21.15.67' 2024-04-22T22:18:44.849 INFO:teuthology.orchestra.run.smithi067.stderr:Deploying unmanaged mon service... 2024-04-22T22:18:45.456 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: Scheduled mon update... 2024-04-22T22:18:45.496 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:45 smithi067 ceph-mon[21875]: Deploying cephadm binary to smithi067 2024-04-22T22:18:45.496 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:45 smithi067 ceph-mon[21875]: from='mgr.14120 172.21.15.67:0/3823221475' entity='mgr.y' 2024-04-22T22:18:45.496 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:45 smithi067 ceph-mon[21875]: Added host smithi067 2024-04-22T22:18:45.496 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:45 smithi067 ceph-mon[21875]: from='mgr.14120 172.21.15.67:0/3823221475' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:18:46.149 INFO:teuthology.orchestra.run.smithi067.stderr:Deploying unmanaged mgr service... 2024-04-22T22:18:46.708 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:46 smithi067 ceph-mon[21875]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:18:46.708 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:46 smithi067 ceph-mon[21875]: Saving service mon spec with placement count:5 2024-04-22T22:18:46.708 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:46 smithi067 ceph-mon[21875]: from='mgr.14120 172.21.15.67:0/3823221475' entity='mgr.y' 2024-04-22T22:18:46.761 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-04-22T22:18:48.099 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:47 smithi067 ceph-mon[21875]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:18:48.099 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:47 smithi067 ceph-mon[21875]: Saving service mgr spec with placement count:2 2024-04-22T22:18:48.099 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:47 smithi067 ceph-mon[21875]: from='mgr.14120 172.21.15.67:0/3823221475' entity='mgr.y' 2024-04-22T22:18:49.496 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:49 smithi067 ceph-mon[21875]: from='mgr.14120 172.21.15.67:0/3823221475' entity='mgr.y' 2024-04-22T22:18:49.497 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:49 smithi067 ceph-mon[21875]: from='mgr.14120 172.21.15.67:0/3823221475' entity='mgr.y' 2024-04-22T22:18:49.497 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:49 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/2173356947' entity='client.admin' 2024-04-22T22:18:50.996 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:50 smithi067 ceph-mon[21875]: from='mgr.14120 172.21.15.67:0/3823221475' entity='mgr.y' 2024-04-22T22:18:50.996 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:50 smithi067 ceph-mon[21875]: from='mgr.14120 172.21.15.67:0/3823221475' entity='mgr.y' 2024-04-22T22:18:50.996 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:50 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/835634311' entity='client.admin' 2024-04-22T22:18:51.267 INFO:teuthology.orchestra.run.smithi067.stderr:Enabling the dashboard module... 2024-04-22T22:18:52.162 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:52 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/3884980124' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-04-22T22:18:52.163 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:52 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ignoring --setuser ceph since I am not root 2024-04-22T22:18:52.163 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:52 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ignoring --setgroup ceph since I am not root 2024-04-22T22:18:52.413 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:52 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:52.198+0000 7f05c9a0c000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-22T22:18:52.413 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:52 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:52.332+0000 7f05c9a0c000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-22T22:18:52.413 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:52 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:52.434+0000 7f05c9a0c000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-22T22:18:52.996 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:52 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:52.754+0000 7f05c9a0c000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-22T22:18:52.997 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:52 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:52.839+0000 7f05c9a0c000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-22T22:18:53.385 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:53 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/3884980124' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-04-22T22:18:53.385 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:53 smithi067 ceph-mon[21875]: mgrmap e9: y(active, since 16s) 2024-04-22T22:18:54.102 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: { 2024-04-22T22:18:54.102 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "epoch": 9, 2024-04-22T22:18:54.102 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "available": true, 2024-04-22T22:18:54.102 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "active_name": "y", 2024-04-22T22:18:54.102 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-22T22:18:54.102 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: } 2024-04-22T22:18:54.271 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:54 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/1437402456' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-22T22:18:54.271 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:54 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:54.031+0000 7f05c9a0c000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-22T22:18:54.271 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:54 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:54.206+0000 7f05c9a0c000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-22T22:18:54.271 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:54 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:54.292+0000 7f05c9a0c000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-22T22:18:54.728 INFO:teuthology.orchestra.run.smithi067.stderr:Waiting for the mgr to restart... 2024-04-22T22:18:54.729 INFO:teuthology.orchestra.run.smithi067.stderr:Waiting for mgr epoch 9... 2024-04-22T22:18:54.851 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:54 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:54.619+0000 7f05c9a0c000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-22T22:18:54.851 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:54 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:54.713+0000 7f05c9a0c000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-22T22:18:54.851 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:54 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:54.815+0000 7f05c9a0c000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-22T22:18:55.178 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:55 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:55.066+0000 7f05c9a0c000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-22T22:18:55.996 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:55 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:55.531+0000 7f05c9a0c000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-22T22:18:56.415 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:56 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:56.032+0000 7f05c9a0c000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-22T22:18:56.416 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:56 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:56.129+0000 7f05c9a0c000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-22T22:18:56.416 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:56 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:56.436+0000 7f05c9a0c000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-22T22:18:56.747 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:56 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:56.529+0000 7f05c9a0c000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-22T22:18:57.246 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:56 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:56.944+0000 7f05c9a0c000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-22T22:18:57.247 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:57 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:57.081+0000 7f05c9a0c000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-22T22:18:57.247 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:57 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:57.176+0000 7f05c9a0c000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-22T22:18:57.746 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:57 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:57.294+0000 7f05c9a0c000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-22T22:18:57.747 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:57 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:57.386+0000 7f05c9a0c000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-22T22:18:58.246 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:57 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:57.955+0000 7f05c9a0c000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-22T22:18:58.555 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:58 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:18:58.325+0000 7f05c9a0c000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-22T22:18:59.307 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:59 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: [22/Apr/2024:22:18:59] ENGINE Bus STARTING 2024-04-22T22:18:59.373 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: { 2024-04-22T22:18:59.373 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-04-22T22:18:59.373 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: "initialized": true 2024-04-22T22:18:59.373 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: } 2024-04-22T22:18:59.562 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:59 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: [22/Apr/2024:22:18:59] ENGINE Serving on https://172.21.15.67:7150 2024-04-22T22:18:59.562 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:18:59 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: [22/Apr/2024:22:18:59] ENGINE Bus STARTED 2024-04-22T22:18:59.563 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:59 smithi067 ceph-mon[21875]: Active manager daemon y restarted 2024-04-22T22:18:59.563 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:59 smithi067 ceph-mon[21875]: Activating manager daemon y 2024-04-22T22:18:59.563 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:59 smithi067 ceph-mon[21875]: osdmap e3: 0 total, 0 up, 0 in 2024-04-22T22:18:59.563 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:59 smithi067 ceph-mon[21875]: mgrmap e10: y(active, starting, since 0.0557005s) 2024-04-22T22:18:59.563 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:59 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T22:18:59.563 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:59 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T22:18:59.563 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:59 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T22:18:59.563 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:59 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T22:18:59.563 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:59 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T22:18:59.563 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:59 smithi067 ceph-mon[21875]: Manager daemon y is now available 2024-04-22T22:18:59.563 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:59 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:18:59.563 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:59 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:18:59.563 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:59 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:18:59.563 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:18:59 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:19:00.147 INFO:teuthology.orchestra.run.smithi067.stderr:mgr epoch 9 is available 2024-04-22T22:19:00.147 INFO:teuthology.orchestra.run.smithi067.stderr:Generating a dashboard self-signed certificate... 2024-04-22T22:19:00.414 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:00 smithi067 ceph-mon[21875]: [22/Apr/2024:22:18:59] ENGINE Bus STARTING 2024-04-22T22:19:00.414 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:00 smithi067 ceph-mon[21875]: [22/Apr/2024:22:18:59] ENGINE Serving on https://172.21.15.67:7150 2024-04-22T22:19:00.415 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:00 smithi067 ceph-mon[21875]: [22/Apr/2024:22:18:59] ENGINE Bus STARTED 2024-04-22T22:19:00.415 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:00 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:00.415 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:00 smithi067 ceph-mon[21875]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-22T22:19:00.415 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:00 smithi067 ceph-mon[21875]: mgrmap e11: y(active, since 1.06074s) 2024-04-22T22:19:00.415 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:00 smithi067 ceph-mon[21875]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-22T22:19:00.415 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:00 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:00.932 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: Self-signed certificate created 2024-04-22T22:19:01.611 INFO:teuthology.orchestra.run.smithi067.stderr:Creating initial admin user... 2024-04-22T22:19:01.844 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:01 smithi067 ceph-mon[21875]: mgrmap e12: y(active, since 2s) 2024-04-22T22:19:01.844 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:01 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:01.844 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:01 smithi067 ceph-mon[21875]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:19:01.844 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:01 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:01.844 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:01 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:02.746 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$IPAZuIFNNK9jn70mXOiyo.lkRpY1w.iS8G4a1KkZPspx0RtonjY0i", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1713824342, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-04-22T22:19:03.742 INFO:teuthology.orchestra.run.smithi067.stderr:Fetching dashboard port number... 2024-04-22T22:19:03.993 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:03 smithi067 ceph-mon[21875]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:19:03.993 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:03 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:04.398 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: 8443 2024-04-22T22:19:04.987 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:04 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/230119788' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-04-22T22:19:05.002 INFO:teuthology.orchestra.run.smithi067.stderr:systemctl: disabled 2024-04-22T22:19:05.010 INFO:teuthology.orchestra.run.smithi067.stderr:systemctl: inactive 2024-04-22T22:19:05.011 INFO:teuthology.orchestra.run.smithi067.stderr:firewalld.service is not enabled 2024-04-22T22:19:05.012 INFO:teuthology.orchestra.run.smithi067.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-04-22T22:19:05.014 INFO:teuthology.orchestra.run.smithi067.stderr:Ceph Dashboard is now available at: 2024-04-22T22:19:05.014 INFO:teuthology.orchestra.run.smithi067.stderr: 2024-04-22T22:19:05.014 INFO:teuthology.orchestra.run.smithi067.stderr: URL: https://smithi067.front.sepia.ceph.com:8443/ 2024-04-22T22:19:05.015 INFO:teuthology.orchestra.run.smithi067.stderr: User: admin 2024-04-22T22:19:05.015 INFO:teuthology.orchestra.run.smithi067.stderr: Password: vabcwb3yc8 2024-04-22T22:19:05.015 INFO:teuthology.orchestra.run.smithi067.stderr: 2024-04-22T22:19:05.015 INFO:teuthology.orchestra.run.smithi067.stderr:Enabling autotune for osd_memory_target 2024-04-22T22:19:05.977 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:05 smithi067 ceph-mon[21875]: mgrmap e13: y(active, since 6s) 2024-04-22T22:19:06.980 INFO:teuthology.orchestra.run.smithi067.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-04-22T22:19:07.594 INFO:teuthology.orchestra.run.smithi067.stderr:You can access the Ceph CLI with: 2024-04-22T22:19:07.594 INFO:teuthology.orchestra.run.smithi067.stderr: 2024-04-22T22:19:07.594 INFO:teuthology.orchestra.run.smithi067.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-04-22T22:19:07.594 INFO:teuthology.orchestra.run.smithi067.stderr: 2024-04-22T22:19:07.594 INFO:teuthology.orchestra.run.smithi067.stderr:Please consider enabling telemetry to help improve Ceph: 2024-04-22T22:19:07.594 INFO:teuthology.orchestra.run.smithi067.stderr: 2024-04-22T22:19:07.594 INFO:teuthology.orchestra.run.smithi067.stderr: ceph telemetry on 2024-04-22T22:19:07.594 INFO:teuthology.orchestra.run.smithi067.stderr: 2024-04-22T22:19:07.594 INFO:teuthology.orchestra.run.smithi067.stderr:For more information see: 2024-04-22T22:19:07.595 INFO:teuthology.orchestra.run.smithi067.stderr: 2024-04-22T22:19:07.595 INFO:teuthology.orchestra.run.smithi067.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-04-22T22:19:07.595 INFO:teuthology.orchestra.run.smithi067.stderr: 2024-04-22T22:19:07.595 INFO:teuthology.orchestra.run.smithi067.stderr:Bootstrap complete. 2024-04-22T22:19:07.632 INFO:tasks.cephadm:Fetching config... 2024-04-22T22:19:07.632 DEBUG:teuthology.orchestra.run.smithi067:> set -ex 2024-04-22T22:19:07.632 DEBUG:teuthology.orchestra.run.smithi067:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-04-22T22:19:07.649 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-04-22T22:19:07.649 DEBUG:teuthology.orchestra.run.smithi067:> set -ex 2024-04-22T22:19:07.649 DEBUG:teuthology.orchestra.run.smithi067:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-04-22T22:19:07.704 INFO:tasks.cephadm:Fetching mon keyring... 2024-04-22T22:19:07.704 DEBUG:teuthology.orchestra.run.smithi067:> set -ex 2024-04-22T22:19:07.704 DEBUG:teuthology.orchestra.run.smithi067:> sudo dd if=/var/lib/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/mon.a/keyring of=/dev/stdout 2024-04-22T22:19:07.769 INFO:tasks.cephadm:Fetching pub ssh key... 2024-04-22T22:19:07.769 DEBUG:teuthology.orchestra.run.smithi067:> set -ex 2024-04-22T22:19:07.769 DEBUG:teuthology.orchestra.run.smithi067:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-04-22T22:19:07.824 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-04-22T22:19:07.824 DEBUG:teuthology.orchestra.run.smithi067:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDTfcnI5qEd4T4D2qoXhOFIouxajTbw4+dH65fR/Qv49JPh05gUQnfNJg0JaWsrcmLTwiUKCKC0U2wLqtNeYEgvI+QyYMtMNO+yrEP4xJr1riBsV71WC4FIYG3+OkNEcNerOx3bW18JsaUmeM7ZfqCzOpe66me/+ukhH+Lsp4VGszKZ+/RSmTKt8mcadLdiqewg0C5WVsYZeE9FoNwbWxqoEKIB2oo8+72wGsOBRjoySFMc3ddtGSqj+JVUjOIjfwTNRGFHoQ71VTyYS3uq1pE8CuI8O+c63fUBpw7FECOMrHZf0w1tIuvVxjns3UKV4teF9/AD4Va1WGN2XqHN2hcgcAqHnRJ7Br/QhX0nfi4L9fVykV9ktRytrT1jBoB8+NF3Xqm4zptscfHd80K/bVSP+47VLmVj4in4yrk+k1aDTzXVdVoYsO5Db2RH11RlXbbKIaXzTSRpJqFsk41Ewp5HxGXtgOCoMcNdFTHJr7uqbCGMHJuQAhf3jIbtfZk+WKE= ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-22T22:19:07.896 INFO:teuthology.orchestra.run.smithi067.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDTfcnI5qEd4T4D2qoXhOFIouxajTbw4+dH65fR/Qv49JPh05gUQnfNJg0JaWsrcmLTwiUKCKC0U2wLqtNeYEgvI+QyYMtMNO+yrEP4xJr1riBsV71WC4FIYG3+OkNEcNerOx3bW18JsaUmeM7ZfqCzOpe66me/+ukhH+Lsp4VGszKZ+/RSmTKt8mcadLdiqewg0C5WVsYZeE9FoNwbWxqoEKIB2oo8+72wGsOBRjoySFMc3ddtGSqj+JVUjOIjfwTNRGFHoQ71VTyYS3uq1pE8CuI8O+c63fUBpw7FECOMrHZf0w1tIuvVxjns3UKV4teF9/AD4Va1WGN2XqHN2hcgcAqHnRJ7Br/QhX0nfi4L9fVykV9ktRytrT1jBoB8+NF3Xqm4zptscfHd80K/bVSP+47VLmVj4in4yrk+k1aDTzXVdVoYsO5Db2RH11RlXbbKIaXzTSRpJqFsk41Ewp5HxGXtgOCoMcNdFTHJr7uqbCGMHJuQAhf3jIbtfZk+WKE= ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968 2024-04-22T22:19:07.907 DEBUG:teuthology.orchestra.run.smithi086:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDTfcnI5qEd4T4D2qoXhOFIouxajTbw4+dH65fR/Qv49JPh05gUQnfNJg0JaWsrcmLTwiUKCKC0U2wLqtNeYEgvI+QyYMtMNO+yrEP4xJr1riBsV71WC4FIYG3+OkNEcNerOx3bW18JsaUmeM7ZfqCzOpe66me/+ukhH+Lsp4VGszKZ+/RSmTKt8mcadLdiqewg0C5WVsYZeE9FoNwbWxqoEKIB2oo8+72wGsOBRjoySFMc3ddtGSqj+JVUjOIjfwTNRGFHoQ71VTyYS3uq1pE8CuI8O+c63fUBpw7FECOMrHZf0w1tIuvVxjns3UKV4teF9/AD4Va1WGN2XqHN2hcgcAqHnRJ7Br/QhX0nfi4L9fVykV9ktRytrT1jBoB8+NF3Xqm4zptscfHd80K/bVSP+47VLmVj4in4yrk+k1aDTzXVdVoYsO5Db2RH11RlXbbKIaXzTSRpJqFsk41Ewp5HxGXtgOCoMcNdFTHJr7uqbCGMHJuQAhf3jIbtfZk+WKE= ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-22T22:19:07.948 INFO:teuthology.orchestra.run.smithi086.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDTfcnI5qEd4T4D2qoXhOFIouxajTbw4+dH65fR/Qv49JPh05gUQnfNJg0JaWsrcmLTwiUKCKC0U2wLqtNeYEgvI+QyYMtMNO+yrEP4xJr1riBsV71WC4FIYG3+OkNEcNerOx3bW18JsaUmeM7ZfqCzOpe66me/+ukhH+Lsp4VGszKZ+/RSmTKt8mcadLdiqewg0C5WVsYZeE9FoNwbWxqoEKIB2oo8+72wGsOBRjoySFMc3ddtGSqj+JVUjOIjfwTNRGFHoQ71VTyYS3uq1pE8CuI8O+c63fUBpw7FECOMrHZf0w1tIuvVxjns3UKV4teF9/AD4Va1WGN2XqHN2hcgcAqHnRJ7Br/QhX0nfi4L9fVykV9ktRytrT1jBoB8+NF3Xqm4zptscfHd80K/bVSP+47VLmVj4in4yrk+k1aDTzXVdVoYsO5Db2RH11RlXbbKIaXzTSRpJqFsk41Ewp5HxGXtgOCoMcNdFTHJr7uqbCGMHJuQAhf3jIbtfZk+WKE= ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968 2024-04-22T22:19:07.960 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-04-22T22:19:08.017 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:07 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/1183204293' entity='client.admin' 2024-04-22T22:19:10.726 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:10 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:10.726 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:10 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi067", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:19:10.726 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:10 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:10.727 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:10 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:19:10.727 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:10 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:11.784 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-04-22T22:19:11.785 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-04-22T22:19:12.201 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:12 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/2174752299' entity='client.admin' 2024-04-22T22:19:12.201 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:12 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:19:12.201 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:12 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:14.416 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi086 2024-04-22T22:19:14.417 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-04-22T22:19:14.417 DEBUG:teuthology.orchestra.run.smithi086:> dd of=/etc/ceph/ceph.conf 2024-04-22T22:19:14.435 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-04-22T22:19:14.435 DEBUG:teuthology.orchestra.run.smithi086:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-22T22:19:14.491 INFO:tasks.cephadm:Adding host smithi086 to orchestrator... 2024-04-22T22:19:14.492 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph orch host add smithi086 2024-04-22T22:19:15.047 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:14 smithi067 ceph-mon[21875]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:19:15.047 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:14 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:15.047 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:14 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:19:15.047 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:14 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:15.047 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:14 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:19:15.047 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:14 smithi067 ceph-mon[21875]: Updating smithi067:/etc/ceph/ceph.conf 2024-04-22T22:19:15.047 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:14 smithi067 ceph-mon[21875]: Updating smithi067:/etc/ceph/ceph.client.admin.keyring 2024-04-22T22:19:15.047 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:14 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:15.048 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:14 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:17.275 INFO:teuthology.orchestra.run.smithi067.stdout:Added host 'smithi086' with addr '172.21.15.86' 2024-04-22T22:19:17.497 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:17 smithi067 ceph-mon[21875]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi086", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:19:17.498 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:17 smithi067 ceph-mon[21875]: Deploying cephadm binary to smithi086 2024-04-22T22:19:17.903 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph orch host ls --format=json 2024-04-22T22:19:18.497 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:18 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:18.498 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:18 smithi067 ceph-mon[21875]: Added host smithi086 2024-04-22T22:19:18.498 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:18 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:19:18.498 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:18 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:18.498 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:18 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:19:18.498 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:18 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:18.498 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:18 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:19.749 INFO:teuthology.orchestra.run.smithi067.stdout: 2024-04-22T22:19:19.749 INFO:teuthology.orchestra.run.smithi067.stdout:[{"addr": "172.21.15.67", "hostname": "smithi067", "labels": [], "status": ""}, {"addr": "172.21.15.86", "hostname": "smithi086", "labels": [], "status": ""}] 2024-04-22T22:19:20.339 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:20 smithi067 ceph-mon[21875]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:20.339 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:20 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:20.365 INFO:tasks.cephadm:Setting crush tunables to default 2024-04-22T22:19:20.366 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph osd crush tunables default 2024-04-22T22:19:21.248 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:21 smithi067 ceph-mon[21875]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T22:19:22.210 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:22 smithi067 ceph-mon[21875]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:22.210 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:22 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:22.210 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:22 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:22.210 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:22 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi086", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:19:22.211 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:22 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:23.068 INFO:teuthology.orchestra.run.smithi067.stderr:adjusted tunables profile to default 2024-04-22T22:19:23.248 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:23 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/3122444316' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-22T22:19:23.622 INFO:tasks.cephadm:Adding mon.a on smithi067 2024-04-22T22:19:23.622 INFO:tasks.cephadm:Adding mon.c on smithi067 2024-04-22T22:19:23.622 INFO:tasks.cephadm:Adding mon.b on smithi086 2024-04-22T22:19:23.622 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph orch apply mon '3;smithi067:172.21.15.67=a;smithi067:[v2:172.21.15.67:3301,v1:172.21.15.67:6790]=c;smithi086:172.21.15.86=b' 2024-04-22T22:19:24.498 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:24 smithi067 ceph-mon[21875]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:24.498 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:24 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/3122444316' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-22T22:19:24.498 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:24 smithi067 ceph-mon[21875]: osdmap e4: 0 total, 0 up, 0 in 2024-04-22T22:19:25.420 INFO:teuthology.orchestra.run.smithi086.stdout:Scheduled mon update... 2024-04-22T22:19:25.748 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:25 smithi067 ceph-mon[21875]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:25.748 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:25 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:25.748 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:25 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:19:25.749 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:25 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:25.749 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:25 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:19:26.142 DEBUG:teuthology.orchestra.run.smithi067:mon.c> sudo journalctl -f -n 0 -u ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@mon.c.service 2024-04-22T22:19:26.145 DEBUG:teuthology.orchestra.run.smithi086:mon.b> sudo journalctl -f -n 0 -u ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@mon.b.service 2024-04-22T22:19:26.148 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-22T22:19:26.148 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-22T22:19:27.097 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:26 smithi067 ceph-mon[21875]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi067:172.21.15.67=a;smithi067:[v2:172.21.15.67:3301,v1:172.21.15.67:6790]=c;smithi086:172.21.15.86=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:19:27.098 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:26 smithi067 ceph-mon[21875]: Saving service mon spec with placement smithi067:172.21.15.67=a;smithi067:[v2:172.21.15.67:3301,v1:172.21.15.67:6790]=c;smithi086:172.21.15.86=b;count:3 2024-04-22T22:19:27.098 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:26 smithi067 ceph-mon[21875]: Updating smithi086:/etc/ceph/ceph.conf 2024-04-22T22:19:27.098 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:26 smithi067 ceph-mon[21875]: Updating smithi086:/etc/ceph/ceph.client.admin.keyring 2024-04-22T22:19:27.098 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:26 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:27.098 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:26 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:27.098 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:26 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T22:19:27.098 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:26 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:27.098 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:26 smithi067 ceph-mon[21875]: Deploying daemon mon.c on smithi067 2024-04-22T22:19:28.065 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-22T22:19:28.065 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":1,"fsid":"1ab1a3a8-00f6-11ef-bc93-c7b262605968","modified":"2024-04-22T22:18:06.454399Z","created":"2024-04-22T22:18:06.454399Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:3300","nonce":0},{"type":"v1","addr":"172.21.15.67:6789","nonce":0}]},"addr":"172.21.15.67:6789/0","public_addr":"172.21.15.67:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-22T22:19:28.066 INFO:teuthology.orchestra.run.smithi086.stderr:dumped monmap epoch 1 2024-04-22T22:19:28.075 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:27 smithi067 ceph-mon[21875]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:28.748 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 podman[26614]: 2024-04-22 22:19:28.683057916 +0000 UTC m=+0.115565902 container create e93c724294c9a6ae8393005e1168e35745b8f405fe7fb7fd3285591a5906b519 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mon-c, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, vcs-type=git, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, RELEASE=HEAD, release=754, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, architecture=x86_64, io.openshift.expose-services=, vendor=Red Hat, Inc., ceph=True, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream) 2024-04-22T22:19:29.248 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[21875]: from='client.? 172.21.15.86:0/4038011321' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T22:19:29.249 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 podman[26614]: 2024-04-22 22:19:28.902061811 +0000 UTC m=+0.334569796 container init e93c724294c9a6ae8393005e1168e35745b8f405fe7fb7fd3285591a5906b519 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mon-c, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, name=centos-stream, vcs-type=git, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, io.openshift.tags=base centos centos-stream, release=754, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, maintainer=Guillaume Abrioux , version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=) 2024-04-22T22:19:29.249 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 podman[26614]: 2024-04-22 22:19:28.905342598 +0000 UTC m=+0.337850586 container start e93c724294c9a6ae8393005e1168e35745b8f405fe7fb7fd3285591a5906b519 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mon-c, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, distribution-scope=public, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, version=8, GIT_CLEAN=True, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-04-22T22:19:29.249 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: set uid:gid to 167:167 (ceph:ceph) 2024-04-22T22:19:29.249 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-04-22T22:19:29.249 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: pidfile_write: ignore empty --pid-file 2024-04-22T22:19:29.249 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: load: jerasure load: lrc 2024-04-22T22:19:29.249 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: RocksDB version: 6.15.5 2024-04-22T22:19:29.249 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-22T22:19:29.250 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Compile date Apr 18 2022 2024-04-22T22:19:29.250 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: DB SUMMARY 2024-04-22T22:19:29.250 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: DB Session ID: FOQN00Q7H15VWD8GW73G 2024-04-22T22:19:29.250 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: CURRENT file: CURRENT 2024-04-22T22:19:29.250 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: IDENTITY file: IDENTITY 2024-04-22T22:19:29.250 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-04-22T22:19:29.250 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-04-22T22:19:29.250 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 511 ; 2024-04-22T22:19:29.250 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.error_if_exists: 0 2024-04-22T22:19:29.250 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.create_if_missing: 0 2024-04-22T22:19:29.250 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.paranoid_checks: 1 2024-04-22T22:19:29.250 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-22T22:19:29.250 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.env: 0x55dac6cc6860 2024-04-22T22:19:29.251 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.fs: Posix File System 2024-04-22T22:19:29.251 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.info_log: 0x55dac8f87dc0 2024-04-22T22:19:29.251 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.max_file_opening_threads: 16 2024-04-22T22:19:29.251 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.statistics: (nil) 2024-04-22T22:19:29.251 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.use_fsync: 0 2024-04-22T22:19:29.251 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.max_log_file_size: 0 2024-04-22T22:19:29.251 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-22T22:19:29.251 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-22T22:19:29.251 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.keep_log_file_num: 1000 2024-04-22T22:19:29.251 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.recycle_log_file_num: 0 2024-04-22T22:19:29.251 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.allow_fallocate: 1 2024-04-22T22:19:29.251 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.allow_mmap_reads: 0 2024-04-22T22:19:29.251 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.allow_mmap_writes: 0 2024-04-22T22:19:29.251 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.use_direct_reads: 0 2024-04-22T22:19:29.252 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-22T22:19:29.252 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.create_missing_column_families: 0 2024-04-22T22:19:29.252 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.db_log_dir: 2024-04-22T22:19:29.252 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2024-04-22T22:19:29.252 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-22T22:19:29.252 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-22T22:19:29.252 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-22T22:19:29.252 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-22T22:19:29.252 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-22T22:19:29.252 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-22T22:19:29.252 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.advise_random_on_open: 1 2024-04-22T22:19:29.252 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.db_write_buffer_size: 0 2024-04-22T22:19:29.252 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.write_buffer_manager: 0x55dac90782d0 2024-04-22T22:19:29.253 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-22T22:19:29.253 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-22T22:19:29.253 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-22T22:19:29.253 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-22T22:19:29.253 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.rate_limiter: (nil) 2024-04-22T22:19:29.253 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-22T22:19:29.253 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.wal_recovery_mode: 2 2024-04-22T22:19:29.253 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.enable_thread_tracking: 0 2024-04-22T22:19:29.253 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.enable_pipelined_write: 0 2024-04-22T22:19:29.253 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.unordered_write: 0 2024-04-22T22:19:29.253 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-22T22:19:29.253 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-22T22:19:29.253 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-22T22:19:29.253 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-22T22:19:29.254 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.row_cache: None 2024-04-22T22:19:29.254 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.wal_filter: None 2024-04-22T22:19:29.254 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-22T22:19:29.254 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.allow_ingest_behind: 0 2024-04-22T22:19:29.254 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.preserve_deletes: 0 2024-04-22T22:19:29.254 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.two_write_queues: 0 2024-04-22T22:19:29.254 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.manual_wal_flush: 0 2024-04-22T22:19:29.254 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.atomic_flush: 0 2024-04-22T22:19:29.254 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-22T22:19:29.254 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-22T22:19:29.254 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-22T22:19:29.254 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.log_readahead_size: 0 2024-04-22T22:19:29.255 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-22T22:19:29.255 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.best_efforts_recovery: 0 2024-04-22T22:19:29.255 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-22T22:19:29.255 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-22T22:19:29.255 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.allow_data_in_errors: 0 2024-04-22T22:19:29.255 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.db_host_id: __hostname__ 2024-04-22T22:19:29.255 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.max_background_jobs: 2 2024-04-22T22:19:29.255 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.max_background_compactions: -1 2024-04-22T22:19:29.255 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.max_subcompactions: 1 2024-04-22T22:19:29.255 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-22T22:19:29.255 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-22T22:19:29.256 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-22T22:19:29.256 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.max_total_wal_size: 0 2024-04-22T22:19:29.256 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-22T22:19:29.257 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-22T22:19:29.257 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-22T22:19:29.257 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-22T22:19:29.257 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.max_open_files: -1 2024-04-22T22:19:29.257 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.bytes_per_sync: 0 2024-04-22T22:19:29.257 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-22T22:19:29.257 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-22T22:19:29.257 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.compaction_readahead_size: 0 2024-04-22T22:19:29.257 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.max_background_flushes: -1 2024-04-22T22:19:29.257 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Compression algorithms supported: 2024-04-22T22:19:29.257 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-22T22:19:29.257 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: kZSTD supported: 0 2024-04-22T22:19:29.257 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: kXpressCompression supported: 0 2024-04-22T22:19:29.257 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: kLZ4HCCompression supported: 1 2024-04-22T22:19:29.257 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: kLZ4Compression supported: 1 2024-04-22T22:19:29.258 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: kBZip2Compression supported: 0 2024-04-22T22:19:29.258 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: kZlibCompression supported: 1 2024-04-22T22:19:29.258 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: kSnappyCompression supported: 1 2024-04-22T22:19:29.258 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-22T22:19:29.258 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2024-04-22T22:19:29.258 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-04-22T22:19:29.258 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-22T22:19:29.258 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.merge_operator: 2024-04-22T22:19:29.258 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.compaction_filter: None 2024-04-22T22:19:29.258 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.compaction_filter_factory: None 2024-04-22T22:19:29.258 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.sst_partitioner_factory: None 2024-04-22T22:19:29.258 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-22T22:19:29.258 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-22T22:19:29.258 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55dac8f53d00) 2024-04-22T22:19:29.258 INFO:journalctl@ceph.mon.c.smithi067.stdout: cache_index_and_filter_blocks: 1 2024-04-22T22:19:29.258 INFO:journalctl@ceph.mon.c.smithi067.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-22T22:19:29.259 INFO:journalctl@ceph.mon.c.smithi067.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-22T22:19:29.259 INFO:journalctl@ceph.mon.c.smithi067.stdout: pin_top_level_index_and_filter: 1 2024-04-22T22:19:29.259 INFO:journalctl@ceph.mon.c.smithi067.stdout: index_type: 0 2024-04-22T22:19:29.259 INFO:journalctl@ceph.mon.c.smithi067.stdout: data_block_index_type: 0 2024-04-22T22:19:29.259 INFO:journalctl@ceph.mon.c.smithi067.stdout: index_shortening: 1 2024-04-22T22:19:29.259 INFO:journalctl@ceph.mon.c.smithi067.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-22T22:19:29.259 INFO:journalctl@ceph.mon.c.smithi067.stdout: hash_index_allow_collision: 1 2024-04-22T22:19:29.259 INFO:journalctl@ceph.mon.c.smithi067.stdout: checksum: 1 2024-04-22T22:19:29.259 INFO:journalctl@ceph.mon.c.smithi067.stdout: no_block_cache: 0 2024-04-22T22:19:29.259 INFO:journalctl@ceph.mon.c.smithi067.stdout: block_cache: 0x55dac8fbe170 2024-04-22T22:19:29.259 INFO:journalctl@ceph.mon.c.smithi067.stdout: block_cache_name: BinnedLRUCache 2024-04-22T22:19:29.259 INFO:journalctl@ceph.mon.c.smithi067.stdout: block_cache_options: 2024-04-22T22:19:29.259 INFO:journalctl@ceph.mon.c.smithi067.stdout: capacity : 536870912 2024-04-22T22:19:29.259 INFO:journalctl@ceph.mon.c.smithi067.stdout: num_shard_bits : 4 2024-04-22T22:19:29.259 INFO:journalctl@ceph.mon.c.smithi067.stdout: strict_capacity_limit : 0 2024-04-22T22:19:29.260 INFO:journalctl@ceph.mon.c.smithi067.stdout: high_pri_pool_ratio: 0.000 2024-04-22T22:19:29.260 INFO:journalctl@ceph.mon.c.smithi067.stdout: block_cache_compressed: (nil) 2024-04-22T22:19:29.260 INFO:journalctl@ceph.mon.c.smithi067.stdout: persistent_cache: (nil) 2024-04-22T22:19:29.260 INFO:journalctl@ceph.mon.c.smithi067.stdout: block_size: 4096 2024-04-22T22:19:29.260 INFO:journalctl@ceph.mon.c.smithi067.stdout: block_size_deviation: 10 2024-04-22T22:19:29.260 INFO:journalctl@ceph.mon.c.smithi067.stdout: block_restart_interval: 16 2024-04-22T22:19:29.260 INFO:journalctl@ceph.mon.c.smithi067.stdout: index_block_restart_interval: 1 2024-04-22T22:19:29.260 INFO:journalctl@ceph.mon.c.smithi067.stdout: metadata_block_size: 4096 2024-04-22T22:19:29.260 INFO:journalctl@ceph.mon.c.smithi067.stdout: partition_filters: 0 2024-04-22T22:19:29.260 INFO:journalctl@ceph.mon.c.smithi067.stdout: use_delta_encoding: 1 2024-04-22T22:19:29.260 INFO:journalctl@ceph.mon.c.smithi067.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-22T22:19:29.260 INFO:journalctl@ceph.mon.c.smithi067.stdout: whole_key_filtering: 1 2024-04-22T22:19:29.260 INFO:journalctl@ceph.mon.c.smithi067.stdout: verify_compression: 0 2024-04-22T22:19:29.260 INFO:journalctl@ceph.mon.c.smithi067.stdout: read_amp_bytes_per_bit: 0 2024-04-22T22:19:29.260 INFO:journalctl@ceph.mon.c.smithi067.stdout: format_version: 4 2024-04-22T22:19:29.260 INFO:journalctl@ceph.mon.c.smithi067.stdout: enable_index_compression: 1 2024-04-22T22:19:29.260 INFO:journalctl@ceph.mon.c.smithi067.stdout: block_align: 0 2024-04-22T22:19:29.261 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.write_buffer_size: 33554432 2024-04-22T22:19:29.261 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.max_write_buffer_number: 2 2024-04-22T22:19:29.261 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.compression: NoCompression 2024-04-22T22:19:29.261 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.bottommost_compression: Disabled 2024-04-22T22:19:29.261 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.prefix_extractor: nullptr 2024-04-22T22:19:29.261 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-22T22:19:29.261 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.num_levels: 7 2024-04-22T22:19:29.261 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-22T22:19:29.261 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-22T22:19:29.261 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-22T22:19:29.261 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-22T22:19:29.261 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-22T22:19:29.261 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-22T22:19:29.262 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-22T22:19:29.262 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-22T22:19:29.262 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-22T22:19:29.262 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-22T22:19:29.262 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-22T22:19:29.262 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.compression_opts.level: 32767 2024-04-22T22:19:29.262 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.compression_opts.strategy: 0 2024-04-22T22:19:29.262 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-22T22:19:29.262 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-22T22:19:29.262 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-22T22:19:29.262 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.compression_opts.enabled: false 2024-04-22T22:19:29.262 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-22T22:19:29.262 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-22T22:19:29.262 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-22T22:19:29.263 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.target_file_size_base: 67108864 2024-04-22T22:19:29.263 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-22T22:19:29.263 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-22T22:19:29.263 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-22T22:19:29.263 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-22T22:19:29.263 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-22T22:19:29.263 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-22T22:19:29.263 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-22T22:19:29.263 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-22T22:19:29.263 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-22T22:19:29.263 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-22T22:19:29.263 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-22T22:19:29.263 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-22T22:19:29.263 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-22T22:19:29.264 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.arena_block_size: 4194304 2024-04-22T22:19:29.264 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-22T22:19:29.264 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-22T22:19:29.264 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-22T22:19:29.264 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.disable_auto_compactions: 0 2024-04-22T22:19:29.264 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-22T22:19:29.264 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-22T22:19:29.264 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-22T22:19:29.264 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-22T22:19:29.264 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-22T22:19:29.264 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-22T22:19:29.264 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-22T22:19:29.264 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-22T22:19:29.264 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-22T22:19:29.264 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-22T22:19:29.265 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.table_properties_collectors: 2024-04-22T22:19:29.265 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.inplace_update_support: 0 2024-04-22T22:19:29.265 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-22T22:19:29.265 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-22T22:19:29.265 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-22T22:19:29.265 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-22T22:19:29.265 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.bloom_locality: 0 2024-04-22T22:19:29.265 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.max_successive_merges: 0 2024-04-22T22:19:29.265 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-22T22:19:29.265 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.paranoid_file_checks: 0 2024-04-22T22:19:29.265 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.force_consistency_checks: 1 2024-04-22T22:19:29.265 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.report_bg_io_stats: 0 2024-04-22T22:19:29.265 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.ttl: 2592000 2024-04-22T22:19:29.265 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-22T22:19:29.266 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.enable_blob_files: false 2024-04-22T22:19:29.266 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.min_blob_size: 0 2024-04-22T22:19:29.266 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.blob_file_size: 268435456 2024-04-22T22:19:29.266 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-22T22:19:29.266 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-22T22:19:29.266 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-22T22:19:29.266 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-22T22:19:29.266 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-04-22T22:19:29.266 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-04-22T22:19:29.266 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713824368931594, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-04-22T22:19:29.266 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-04-22T22:19:29.266 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-22T22:19:29.266 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713824368931922, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1713824368, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "369b2a4d-760f-4cc4-a822-35dac5de9402", "db_session_id": "FOQN00Q7H15VWD8GW73G"}} 2024-04-22T22:19:29.267 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-04-22T22:19:29.267 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713824368932257, "job": 1, "event": "recovery_finished"} 2024-04-22T22:19:29.267 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-22T22:19:29.267 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55dac8fa4a80 2024-04-22T22:19:29.267 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: DB pointer 0x55dac8fb4000 2024-04-22T22:19:29.267 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-04-22T22:19:29.267 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-22T22:19:29.267 INFO:journalctl@ceph.mon.c.smithi067.stdout: ** DB Stats ** 2024-04-22T22:19:29.267 INFO:journalctl@ceph.mon.c.smithi067.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T22:19:29.267 INFO:journalctl@ceph.mon.c.smithi067.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-22T22:19:29.267 INFO:journalctl@ceph.mon.c.smithi067.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-22T22:19:29.267 INFO:journalctl@ceph.mon.c.smithi067.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T22:19:29.267 INFO:journalctl@ceph.mon.c.smithi067.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-22T22:19:29.267 INFO:journalctl@ceph.mon.c.smithi067.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-22T22:19:29.268 INFO:journalctl@ceph.mon.c.smithi067.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T22:19:29.268 INFO:journalctl@ceph.mon.c.smithi067.stdout: 2024-04-22T22:19:29.268 INFO:journalctl@ceph.mon.c.smithi067.stdout: ** Compaction Stats [default] ** 2024-04-22T22:19:29.268 INFO:journalctl@ceph.mon.c.smithi067.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T22:19:29.268 INFO:journalctl@ceph.mon.c.smithi067.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T22:19:29.268 INFO:journalctl@ceph.mon.c.smithi067.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.7 0.00 0.00 1 0.000 0 0 2024-04-22T22:19:29.268 INFO:journalctl@ceph.mon.c.smithi067.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.7 0.00 0.00 1 0.000 0 0 2024-04-22T22:19:29.268 INFO:journalctl@ceph.mon.c.smithi067.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.7 0.00 0.00 1 0.000 0 0 2024-04-22T22:19:29.268 INFO:journalctl@ceph.mon.c.smithi067.stdout: 2024-04-22T22:19:29.268 INFO:journalctl@ceph.mon.c.smithi067.stdout: ** Compaction Stats [default] ** 2024-04-22T22:19:29.268 INFO:journalctl@ceph.mon.c.smithi067.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T22:19:29.268 INFO:journalctl@ceph.mon.c.smithi067.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T22:19:29.268 INFO:journalctl@ceph.mon.c.smithi067.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.7 0.00 0.00 1 0.000 0 0 2024-04-22T22:19:29.268 INFO:journalctl@ceph.mon.c.smithi067.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T22:19:29.268 INFO:journalctl@ceph.mon.c.smithi067.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-22T22:19:29.269 INFO:journalctl@ceph.mon.c.smithi067.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-22T22:19:29.269 INFO:journalctl@ceph.mon.c.smithi067.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-22T22:19:29.269 INFO:journalctl@ceph.mon.c.smithi067.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-22T22:19:29.269 INFO:journalctl@ceph.mon.c.smithi067.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-22T22:19:29.269 INFO:journalctl@ceph.mon.c.smithi067.stdout: Cumulative compaction: 0.00 GB write, 0.56 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T22:19:29.269 INFO:journalctl@ceph.mon.c.smithi067.stdout: Interval compaction: 0.00 GB write, 0.56 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T22:19:29.269 INFO:journalctl@ceph.mon.c.smithi067.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-22T22:19:29.269 INFO:journalctl@ceph.mon.c.smithi067.stdout: 2024-04-22T22:19:29.269 INFO:journalctl@ceph.mon.c.smithi067.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-22T22:19:29.269 INFO:journalctl@ceph.mon.c.smithi067.stdout: 2024-04-22T22:19:29.269 INFO:journalctl@ceph.mon.c.smithi067.stdout: ** Compaction Stats [default] ** 2024-04-22T22:19:29.269 INFO:journalctl@ceph.mon.c.smithi067.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T22:19:29.269 INFO:journalctl@ceph.mon.c.smithi067.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T22:19:29.269 INFO:journalctl@ceph.mon.c.smithi067.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.7 0.00 0.00 1 0.000 0 0 2024-04-22T22:19:29.269 INFO:journalctl@ceph.mon.c.smithi067.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.7 0.00 0.00 1 0.000 0 0 2024-04-22T22:19:29.269 INFO:journalctl@ceph.mon.c.smithi067.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-22T22:19:29.270 INFO:journalctl@ceph.mon.c.smithi067.stdout: 2024-04-22T22:19:29.270 INFO:journalctl@ceph.mon.c.smithi067.stdout: ** Compaction Stats [default] ** 2024-04-22T22:19:29.270 INFO:journalctl@ceph.mon.c.smithi067.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T22:19:29.270 INFO:journalctl@ceph.mon.c.smithi067.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T22:19:29.270 INFO:journalctl@ceph.mon.c.smithi067.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.7 0.00 0.00 1 0.000 0 0 2024-04-22T22:19:29.270 INFO:journalctl@ceph.mon.c.smithi067.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T22:19:29.270 INFO:journalctl@ceph.mon.c.smithi067.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-22T22:19:29.270 INFO:journalctl@ceph.mon.c.smithi067.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-22T22:19:29.270 INFO:journalctl@ceph.mon.c.smithi067.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-22T22:19:29.270 INFO:journalctl@ceph.mon.c.smithi067.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-22T22:19:29.270 INFO:journalctl@ceph.mon.c.smithi067.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-22T22:19:29.270 INFO:journalctl@ceph.mon.c.smithi067.stdout: Cumulative compaction: 0.00 GB write, 0.56 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T22:19:29.270 INFO:journalctl@ceph.mon.c.smithi067.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T22:19:29.270 INFO:journalctl@ceph.mon.c.smithi067.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-22T22:19:29.271 INFO:journalctl@ceph.mon.c.smithi067.stdout: 2024-04-22T22:19:29.271 INFO:journalctl@ceph.mon.c.smithi067.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-22T22:19:29.271 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-04-22T22:19:29.271 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: using public_addrv [v2:172.21.15.67:3301/0,v1:172.21.15.67:6790/0] 2024-04-22T22:19:29.271 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: starting mon.c rank -1 at public addrs [v2:172.21.15.67:3301/0,v1:172.21.15.67:6790/0] at bind addrs [v2:172.21.15.67:3301/0,v1:172.21.15.67:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 2024-04-22T22:19:29.271 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: mon.c@-1(???) e0 preinit fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 2024-04-22T22:19:29.271 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: mon.c@-1(synchronizing).mds e1 new map 2024-04-22T22:19:29.271 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: mon.c@-1(synchronizing).mds e1 print_map 2024-04-22T22:19:29.271 INFO:journalctl@ceph.mon.c.smithi067.stdout: e1 2024-04-22T22:19:29.271 INFO:journalctl@ceph.mon.c.smithi067.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-22T22:19:29.271 INFO:journalctl@ceph.mon.c.smithi067.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-22T22:19:29.271 INFO:journalctl@ceph.mon.c.smithi067.stdout: legacy client fscid: -1 2024-04-22T22:19:29.271 INFO:journalctl@ceph.mon.c.smithi067.stdout: 2024-04-22T22:19:29.272 INFO:journalctl@ceph.mon.c.smithi067.stdout: No filesystems configured 2024-04-22T22:19:29.272 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-04-22T22:19:29.272 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-04-22T22:19:29.272 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-04-22T22:19:29.272 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-04-22T22:19:29.272 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-04-22T22:19:29.272 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-04-22T22:19:29.272 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-22T22:19:29.272 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-22T22:19:29.272 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-22T22:19:29.272 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-22T22:19:29.272 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi067:172.21.15.67=a;smithi067:[v2:172.21.15.67:3301,v1:172.21.15.67:6790]=c;smithi086:172.21.15.86=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:19:29.272 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: Saving service mon spec with placement smithi067:172.21.15.67=a;smithi067:[v2:172.21.15.67:3301,v1:172.21.15.67:6790]=c;smithi086:172.21.15.86=b;count:3 2024-04-22T22:19:29.272 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: Updating smithi086:/etc/ceph/ceph.conf 2024-04-22T22:19:29.272 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: Updating smithi086:/etc/ceph/ceph.client.admin.keyring 2024-04-22T22:19:29.272 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:29.273 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:29.273 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T22:19:29.273 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:29.273 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: Deploying daemon mon.c on smithi067 2024-04-22T22:19:29.273 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:29.273 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: from='client.? 172.21.15.86:0/4038011321' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T22:19:29.273 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-04-22T22:19:29.273 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 ceph-mon[26633]: mon.c@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-04-22T22:19:29.273 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 bash[26614]: e93c724294c9a6ae8393005e1168e35745b8f405fe7fb7fd3285591a5906b519 2024-04-22T22:19:29.273 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:28 smithi067 systemd[1]: Started Ceph mon.c for 1ab1a3a8-00f6-11ef-bc93-c7b262605968. 2024-04-22T22:19:29.621 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-22T22:19:29.621 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-22T22:19:34.378 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-22T22:19:34.378 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":2,"fsid":"1ab1a3a8-00f6-11ef-bc93-c7b262605968","modified":"2024-04-22T22:19:28.950414Z","created":"2024-04-22T22:18:06.454399Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:3300","nonce":0},{"type":"v1","addr":"172.21.15.67:6789","nonce":0}]},"addr":"172.21.15.67:6789/0","public_addr":"172.21.15.67:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:3301","nonce":0},{"type":"v1","addr":"172.21.15.67:6790","nonce":0}]},"addr":"172.21.15.67:6790/0","public_addr":"172.21.15.67:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-22T22:19:34.380 INFO:teuthology.orchestra.run.smithi086.stderr:dumped monmap epoch 2 2024-04-22T22:19:34.499 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[21875]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:34.499 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T22:19:34.499 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[21875]: mon.a calling monitor election 2024-04-22T22:19:34.499 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T22:19:34.499 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T22:19:34.499 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[21875]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:34.499 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T22:19:34.499 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[21875]: mon.c calling monitor election 2024-04-22T22:19:34.499 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T22:19:34.499 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[21875]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:34.500 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T22:19:34.500 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T22:19:34.500 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[21875]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-22T22:19:34.500 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[21875]: monmap e2: 2 mons at {a=[v2:172.21.15.67:3300/0,v1:172.21.15.67:6789/0],c=[v2:172.21.15.67:3301/0,v1:172.21.15.67:6790/0]} 2024-04-22T22:19:34.500 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[21875]: fsmap 2024-04-22T22:19:34.500 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[21875]: osdmap e4: 0 total, 0 up, 0 in 2024-04-22T22:19:34.500 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[21875]: mgrmap e13: y(active, since 35s) 2024-04-22T22:19:34.500 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[21875]: overall HEALTH_OK 2024-04-22T22:19:34.500 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:34.500 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T22:19:34.500 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:34.500 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[26633]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:34.500 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T22:19:34.501 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[26633]: mon.a calling monitor election 2024-04-22T22:19:34.501 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T22:19:34.501 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T22:19:34.501 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[26633]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:34.501 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T22:19:34.501 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[26633]: mon.c calling monitor election 2024-04-22T22:19:34.501 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T22:19:34.501 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[26633]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:34.501 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T22:19:34.501 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T22:19:34.501 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[26633]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-22T22:19:34.501 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[26633]: monmap e2: 2 mons at {a=[v2:172.21.15.67:3300/0,v1:172.21.15.67:6789/0],c=[v2:172.21.15.67:3301/0,v1:172.21.15.67:6790/0]} 2024-04-22T22:19:34.501 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[26633]: fsmap 2024-04-22T22:19:34.501 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[26633]: osdmap e4: 0 total, 0 up, 0 in 2024-04-22T22:19:34.501 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[26633]: mgrmap e13: y(active, since 35s) 2024-04-22T22:19:34.501 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[26633]: overall HEALTH_OK 2024-04-22T22:19:34.502 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:34.502 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T22:19:34.502 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:34 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:35.498 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:35 smithi067 ceph-mon[21875]: Deploying daemon mon.b on smithi086 2024-04-22T22:19:35.498 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:35 smithi067 ceph-mon[21875]: from='client.? 172.21.15.86:0/3980945201' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T22:19:35.498 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:35 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T22:19:35.499 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:35 smithi067 ceph-mon[26633]: Deploying daemon mon.b on smithi086 2024-04-22T22:19:35.499 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:35 smithi067 ceph-mon[26633]: from='client.? 172.21.15.86:0/3980945201' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T22:19:35.499 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:35 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T22:19:36.008 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-22T22:19:36.008 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-22T22:19:36.498 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:36 smithi067 ceph-mon[21875]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:36.499 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:36 smithi067 ceph-mon[26633]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:37.747 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:37 smithi086 systemd[1]: Starting Ceph mon.b for 1ab1a3a8-00f6-11ef-bc93-c7b262605968... 2024-04-22T22:19:38.280 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 podman[22622]: 2024-04-22 22:19:38.102700557 +0000 UTC m=+0.027913000 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-22T22:19:38.498 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:38 smithi067 ceph-mon[21875]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:38.499 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:38 smithi067 ceph-mon[26633]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:38.548 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 podman[22622]: 2024-04-22 22:19:38.484006328 +0000 UTC m=+0.409218747 container create be2e0891a57742b32181076130b29486a006138ea30755751c260c73b148f9f4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mon-b, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, name=centos-stream, architecture=x86_64, release=754, io.openshift.expose-services=, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, GIT_BRANCH=HEAD, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-04-22T22:19:39.198 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 podman[22622]: 2024-04-22 22:19:38.820110306 +0000 UTC m=+0.745322728 container init be2e0891a57742b32181076130b29486a006138ea30755751c260c73b148f9f4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mon-b, io.buildah.version=1.19.8, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, architecture=x86_64, version=8, release=754, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD) 2024-04-22T22:19:39.198 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 podman[22622]: 2024-04-22 22:19:38.823547835 +0000 UTC m=+0.748760254 container start be2e0891a57742b32181076130b29486a006138ea30755751c260c73b148f9f4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mon-b, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, version=8, io.openshift.tags=base centos centos-stream, vcs-type=git, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, vendor=Red Hat, Inc., ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754) 2024-04-22T22:19:39.198 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: set uid:gid to 167:167 (ceph:ceph) 2024-04-22T22:19:39.198 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-04-22T22:19:39.198 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: pidfile_write: ignore empty --pid-file 2024-04-22T22:19:39.198 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: load: jerasure load: lrc 2024-04-22T22:19:39.199 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: RocksDB version: 6.15.5 2024-04-22T22:19:39.199 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-22T22:19:39.199 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Compile date Apr 18 2022 2024-04-22T22:19:39.199 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: DB SUMMARY 2024-04-22T22:19:39.199 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: DB Session ID: B27NOIA0X8W03KTKBKJ4 2024-04-22T22:19:39.199 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: CURRENT file: CURRENT 2024-04-22T22:19:39.199 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: IDENTITY file: IDENTITY 2024-04-22T22:19:39.200 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-04-22T22:19:39.200 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2024-04-22T22:19:39.200 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 636 ; 2024-04-22T22:19:39.200 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.error_if_exists: 0 2024-04-22T22:19:39.200 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.create_if_missing: 0 2024-04-22T22:19:39.200 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.paranoid_checks: 1 2024-04-22T22:19:39.200 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-22T22:19:39.200 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.env: 0x55c1320c6860 2024-04-22T22:19:39.200 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.fs: Posix File System 2024-04-22T22:19:39.200 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.info_log: 0x55c134565dc0 2024-04-22T22:19:39.200 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.max_file_opening_threads: 16 2024-04-22T22:19:39.200 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.statistics: (nil) 2024-04-22T22:19:39.200 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.use_fsync: 0 2024-04-22T22:19:39.201 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.max_log_file_size: 0 2024-04-22T22:19:39.201 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-22T22:19:39.201 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-22T22:19:39.201 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.keep_log_file_num: 1000 2024-04-22T22:19:39.201 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.recycle_log_file_num: 0 2024-04-22T22:19:39.201 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.allow_fallocate: 1 2024-04-22T22:19:39.201 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.allow_mmap_reads: 0 2024-04-22T22:19:39.201 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.allow_mmap_writes: 0 2024-04-22T22:19:39.201 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.use_direct_reads: 0 2024-04-22T22:19:39.201 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-22T22:19:39.201 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.create_missing_column_families: 0 2024-04-22T22:19:39.201 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.db_log_dir: 2024-04-22T22:19:39.201 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2024-04-22T22:19:39.201 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-22T22:19:39.202 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-22T22:19:39.202 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-22T22:19:39.202 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-22T22:19:39.202 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-22T22:19:39.202 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-22T22:19:39.202 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.advise_random_on_open: 1 2024-04-22T22:19:39.202 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.db_write_buffer_size: 0 2024-04-22T22:19:39.202 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.write_buffer_manager: 0x55c134656240 2024-04-22T22:19:39.202 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-22T22:19:39.202 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-22T22:19:39.202 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-22T22:19:39.202 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-22T22:19:39.202 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.rate_limiter: (nil) 2024-04-22T22:19:39.202 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-22T22:19:39.203 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.wal_recovery_mode: 2 2024-04-22T22:19:39.203 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.enable_thread_tracking: 0 2024-04-22T22:19:39.203 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.enable_pipelined_write: 0 2024-04-22T22:19:39.203 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.unordered_write: 0 2024-04-22T22:19:39.203 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-22T22:19:39.203 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-22T22:19:39.203 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-22T22:19:39.203 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-22T22:19:39.203 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.row_cache: None 2024-04-22T22:19:39.203 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.wal_filter: None 2024-04-22T22:19:39.203 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-22T22:19:39.203 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.allow_ingest_behind: 0 2024-04-22T22:19:39.203 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.preserve_deletes: 0 2024-04-22T22:19:39.203 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.two_write_queues: 0 2024-04-22T22:19:39.204 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.manual_wal_flush: 0 2024-04-22T22:19:39.204 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.atomic_flush: 0 2024-04-22T22:19:39.204 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-22T22:19:39.204 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-22T22:19:39.204 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-22T22:19:39.204 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.log_readahead_size: 0 2024-04-22T22:19:39.204 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-22T22:19:39.204 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.best_efforts_recovery: 0 2024-04-22T22:19:39.204 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-22T22:19:39.204 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-22T22:19:39.204 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.allow_data_in_errors: 0 2024-04-22T22:19:39.204 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.db_host_id: __hostname__ 2024-04-22T22:19:39.205 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.max_background_jobs: 2 2024-04-22T22:19:39.205 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.max_background_compactions: -1 2024-04-22T22:19:39.205 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.max_subcompactions: 1 2024-04-22T22:19:39.205 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-22T22:19:39.205 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-22T22:19:39.205 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-22T22:19:39.205 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.max_total_wal_size: 0 2024-04-22T22:19:39.205 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-22T22:19:39.205 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-22T22:19:39.205 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-22T22:19:39.205 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-22T22:19:39.205 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.max_open_files: -1 2024-04-22T22:19:39.205 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.bytes_per_sync: 0 2024-04-22T22:19:39.205 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-22T22:19:39.205 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-22T22:19:39.206 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.compaction_readahead_size: 0 2024-04-22T22:19:39.206 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.max_background_flushes: -1 2024-04-22T22:19:39.206 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Compression algorithms supported: 2024-04-22T22:19:39.206 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-22T22:19:39.206 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: kZSTD supported: 0 2024-04-22T22:19:39.206 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: kXpressCompression supported: 0 2024-04-22T22:19:39.206 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: kLZ4HCCompression supported: 1 2024-04-22T22:19:39.206 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: kLZ4Compression supported: 1 2024-04-22T22:19:39.206 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: kBZip2Compression supported: 0 2024-04-22T22:19:39.206 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: kZlibCompression supported: 1 2024-04-22T22:19:39.206 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: kSnappyCompression supported: 1 2024-04-22T22:19:39.206 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-22T22:19:39.206 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2024-04-22T22:19:39.206 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-04-22T22:19:39.207 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-22T22:19:39.207 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.merge_operator: 2024-04-22T22:19:39.207 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.compaction_filter: None 2024-04-22T22:19:39.207 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.compaction_filter_factory: None 2024-04-22T22:19:39.207 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.sst_partitioner_factory: None 2024-04-22T22:19:39.207 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-22T22:19:39.207 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-22T22:19:39.207 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55c134531d00) 2024-04-22T22:19:39.207 INFO:journalctl@ceph.mon.b.smithi086.stdout: cache_index_and_filter_blocks: 1 2024-04-22T22:19:39.207 INFO:journalctl@ceph.mon.b.smithi086.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-22T22:19:39.207 INFO:journalctl@ceph.mon.b.smithi086.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-22T22:19:39.207 INFO:journalctl@ceph.mon.b.smithi086.stdout: pin_top_level_index_and_filter: 1 2024-04-22T22:19:39.207 INFO:journalctl@ceph.mon.b.smithi086.stdout: index_type: 0 2024-04-22T22:19:39.207 INFO:journalctl@ceph.mon.b.smithi086.stdout: data_block_index_type: 0 2024-04-22T22:19:39.207 INFO:journalctl@ceph.mon.b.smithi086.stdout: index_shortening: 1 2024-04-22T22:19:39.207 INFO:journalctl@ceph.mon.b.smithi086.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-22T22:19:39.208 INFO:journalctl@ceph.mon.b.smithi086.stdout: hash_index_allow_collision: 1 2024-04-22T22:19:39.208 INFO:journalctl@ceph.mon.b.smithi086.stdout: checksum: 1 2024-04-22T22:19:39.208 INFO:journalctl@ceph.mon.b.smithi086.stdout: no_block_cache: 0 2024-04-22T22:19:39.208 INFO:journalctl@ceph.mon.b.smithi086.stdout: block_cache: 0x55c13459c170 2024-04-22T22:19:39.209 INFO:journalctl@ceph.mon.b.smithi086.stdout: block_cache_name: BinnedLRUCache 2024-04-22T22:19:39.209 INFO:journalctl@ceph.mon.b.smithi086.stdout: block_cache_options: 2024-04-22T22:19:39.209 INFO:journalctl@ceph.mon.b.smithi086.stdout: capacity : 536870912 2024-04-22T22:19:39.210 INFO:journalctl@ceph.mon.b.smithi086.stdout: num_shard_bits : 4 2024-04-22T22:19:39.210 INFO:journalctl@ceph.mon.b.smithi086.stdout: strict_capacity_limit : 0 2024-04-22T22:19:39.210 INFO:journalctl@ceph.mon.b.smithi086.stdout: high_pri_pool_ratio: 0.000 2024-04-22T22:19:39.210 INFO:journalctl@ceph.mon.b.smithi086.stdout: block_cache_compressed: (nil) 2024-04-22T22:19:39.210 INFO:journalctl@ceph.mon.b.smithi086.stdout: persistent_cache: (nil) 2024-04-22T22:19:39.210 INFO:journalctl@ceph.mon.b.smithi086.stdout: block_size: 4096 2024-04-22T22:19:39.210 INFO:journalctl@ceph.mon.b.smithi086.stdout: block_size_deviation: 10 2024-04-22T22:19:39.210 INFO:journalctl@ceph.mon.b.smithi086.stdout: block_restart_interval: 16 2024-04-22T22:19:39.210 INFO:journalctl@ceph.mon.b.smithi086.stdout: index_block_restart_interval: 1 2024-04-22T22:19:39.210 INFO:journalctl@ceph.mon.b.smithi086.stdout: metadata_block_size: 4096 2024-04-22T22:19:39.210 INFO:journalctl@ceph.mon.b.smithi086.stdout: partition_filters: 0 2024-04-22T22:19:39.210 INFO:journalctl@ceph.mon.b.smithi086.stdout: use_delta_encoding: 1 2024-04-22T22:19:39.210 INFO:journalctl@ceph.mon.b.smithi086.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-22T22:19:39.210 INFO:journalctl@ceph.mon.b.smithi086.stdout: whole_key_filtering: 1 2024-04-22T22:19:39.210 INFO:journalctl@ceph.mon.b.smithi086.stdout: verify_compression: 0 2024-04-22T22:19:39.211 INFO:journalctl@ceph.mon.b.smithi086.stdout: read_amp_bytes_per_bit: 0 2024-04-22T22:19:39.211 INFO:journalctl@ceph.mon.b.smithi086.stdout: format_version: 4 2024-04-22T22:19:39.211 INFO:journalctl@ceph.mon.b.smithi086.stdout: enable_index_compression: 1 2024-04-22T22:19:39.211 INFO:journalctl@ceph.mon.b.smithi086.stdout: block_align: 0 2024-04-22T22:19:39.211 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.write_buffer_size: 33554432 2024-04-22T22:19:39.211 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.max_write_buffer_number: 2 2024-04-22T22:19:39.211 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.compression: NoCompression 2024-04-22T22:19:39.211 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.bottommost_compression: Disabled 2024-04-22T22:19:39.211 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.prefix_extractor: nullptr 2024-04-22T22:19:39.211 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-22T22:19:39.211 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.num_levels: 7 2024-04-22T22:19:39.211 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-22T22:19:39.211 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-22T22:19:39.211 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-22T22:19:39.211 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-22T22:19:39.212 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-22T22:19:39.212 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-22T22:19:39.212 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-22T22:19:39.212 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-22T22:19:39.212 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-22T22:19:39.212 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-22T22:19:39.212 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-22T22:19:39.212 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.compression_opts.level: 32767 2024-04-22T22:19:39.212 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.compression_opts.strategy: 0 2024-04-22T22:19:39.212 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-22T22:19:39.212 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-22T22:19:39.212 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-22T22:19:39.212 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.compression_opts.enabled: false 2024-04-22T22:19:39.212 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-22T22:19:39.212 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-22T22:19:39.212 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-22T22:19:39.213 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.target_file_size_base: 67108864 2024-04-22T22:19:39.213 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-22T22:19:39.213 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-22T22:19:39.213 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-22T22:19:39.213 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-22T22:19:39.213 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-22T22:19:39.213 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-22T22:19:39.213 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-22T22:19:39.213 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-22T22:19:39.213 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-22T22:19:39.213 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-22T22:19:39.213 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-22T22:19:39.213 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-22T22:19:39.213 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-22T22:19:39.213 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.arena_block_size: 4194304 2024-04-22T22:19:39.214 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-22T22:19:39.214 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-22T22:19:39.214 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-22T22:19:39.214 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.disable_auto_compactions: 0 2024-04-22T22:19:39.214 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-22T22:19:39.214 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-22T22:19:39.214 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-22T22:19:39.214 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-22T22:19:39.214 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-22T22:19:39.214 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-22T22:19:39.214 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-22T22:19:39.214 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-22T22:19:39.214 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-22T22:19:39.214 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-22T22:19:39.214 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.table_properties_collectors: 2024-04-22T22:19:39.214 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.inplace_update_support: 0 2024-04-22T22:19:39.215 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-22T22:19:39.215 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-22T22:19:39.215 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-22T22:19:39.215 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-22T22:19:39.215 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.bloom_locality: 0 2024-04-22T22:19:39.215 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.max_successive_merges: 0 2024-04-22T22:19:39.215 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-22T22:19:39.215 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.paranoid_file_checks: 0 2024-04-22T22:19:39.215 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.force_consistency_checks: 1 2024-04-22T22:19:39.215 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.report_bg_io_stats: 0 2024-04-22T22:19:39.215 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.ttl: 2592000 2024-04-22T22:19:39.215 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-22T22:19:39.215 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.enable_blob_files: false 2024-04-22T22:19:39.215 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.min_blob_size: 0 2024-04-22T22:19:39.215 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.blob_file_size: 268435456 2024-04-22T22:19:39.216 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-22T22:19:39.216 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-22T22:19:39.216 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-22T22:19:39.216 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-22T22:19:39.216 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-04-22T22:19:39.216 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-04-22T22:19:39.216 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713824378849444, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-04-22T22:19:39.216 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-04-22T22:19:39.216 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-22T22:19:39.216 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713824378849735, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1713824378, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "9304ce72-f774-483f-8433-53d53a3c73e2", "db_session_id": "B27NOIA0X8W03KTKBKJ4"}} 2024-04-22T22:19:39.216 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-04-22T22:19:39.216 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713824378850045, "job": 1, "event": "recovery_finished"} 2024-04-22T22:19:39.216 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-22T22:19:39.216 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55c134582a80 2024-04-22T22:19:39.216 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: DB pointer 0x55c134592000 2024-04-22T22:19:39.216 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-04-22T22:19:39.217 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-22T22:19:39.217 INFO:journalctl@ceph.mon.b.smithi086.stdout: ** DB Stats ** 2024-04-22T22:19:39.217 INFO:journalctl@ceph.mon.b.smithi086.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T22:19:39.217 INFO:journalctl@ceph.mon.b.smithi086.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-22T22:19:39.217 INFO:journalctl@ceph.mon.b.smithi086.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-22T22:19:39.217 INFO:journalctl@ceph.mon.b.smithi086.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T22:19:39.217 INFO:journalctl@ceph.mon.b.smithi086.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-22T22:19:39.217 INFO:journalctl@ceph.mon.b.smithi086.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-22T22:19:39.217 INFO:journalctl@ceph.mon.b.smithi086.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-22T22:19:39.217 INFO:journalctl@ceph.mon.b.smithi086.stdout: 2024-04-22T22:19:39.217 INFO:journalctl@ceph.mon.b.smithi086.stdout: ** Compaction Stats [default] ** 2024-04-22T22:19:39.217 INFO:journalctl@ceph.mon.b.smithi086.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T22:19:39.217 INFO:journalctl@ceph.mon.b.smithi086.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T22:19:39.217 INFO:journalctl@ceph.mon.b.smithi086.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.8 0.00 0.00 1 0.000 0 0 2024-04-22T22:19:39.217 INFO:journalctl@ceph.mon.b.smithi086.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.8 0.00 0.00 1 0.000 0 0 2024-04-22T22:19:39.217 INFO:journalctl@ceph.mon.b.smithi086.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.8 0.00 0.00 1 0.000 0 0 2024-04-22T22:19:39.218 INFO:journalctl@ceph.mon.b.smithi086.stdout: 2024-04-22T22:19:39.218 INFO:journalctl@ceph.mon.b.smithi086.stdout: ** Compaction Stats [default] ** 2024-04-22T22:19:39.218 INFO:journalctl@ceph.mon.b.smithi086.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T22:19:39.218 INFO:journalctl@ceph.mon.b.smithi086.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T22:19:39.218 INFO:journalctl@ceph.mon.b.smithi086.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.8 0.00 0.00 1 0.000 0 0 2024-04-22T22:19:39.218 INFO:journalctl@ceph.mon.b.smithi086.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T22:19:39.218 INFO:journalctl@ceph.mon.b.smithi086.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-22T22:19:39.218 INFO:journalctl@ceph.mon.b.smithi086.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-22T22:19:39.218 INFO:journalctl@ceph.mon.b.smithi086.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-22T22:19:39.218 INFO:journalctl@ceph.mon.b.smithi086.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-22T22:19:39.218 INFO:journalctl@ceph.mon.b.smithi086.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-22T22:19:39.218 INFO:journalctl@ceph.mon.b.smithi086.stdout: Cumulative compaction: 0.00 GB write, 0.64 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T22:19:39.218 INFO:journalctl@ceph.mon.b.smithi086.stdout: Interval compaction: 0.00 GB write, 0.64 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T22:19:39.218 INFO:journalctl@ceph.mon.b.smithi086.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-22T22:19:39.218 INFO:journalctl@ceph.mon.b.smithi086.stdout: 2024-04-22T22:19:39.219 INFO:journalctl@ceph.mon.b.smithi086.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-22T22:19:39.219 INFO:journalctl@ceph.mon.b.smithi086.stdout: 2024-04-22T22:19:39.219 INFO:journalctl@ceph.mon.b.smithi086.stdout: ** Compaction Stats [default] ** 2024-04-22T22:19:39.219 INFO:journalctl@ceph.mon.b.smithi086.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T22:19:39.219 INFO:journalctl@ceph.mon.b.smithi086.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T22:19:39.219 INFO:journalctl@ceph.mon.b.smithi086.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.8 0.00 0.00 1 0.000 0 0 2024-04-22T22:19:39.219 INFO:journalctl@ceph.mon.b.smithi086.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.8 0.00 0.00 1 0.000 0 0 2024-04-22T22:19:39.219 INFO:journalctl@ceph.mon.b.smithi086.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-22T22:19:39.219 INFO:journalctl@ceph.mon.b.smithi086.stdout: 2024-04-22T22:19:39.219 INFO:journalctl@ceph.mon.b.smithi086.stdout: ** Compaction Stats [default] ** 2024-04-22T22:19:39.219 INFO:journalctl@ceph.mon.b.smithi086.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-22T22:19:39.219 INFO:journalctl@ceph.mon.b.smithi086.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-22T22:19:39.219 INFO:journalctl@ceph.mon.b.smithi086.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.8 0.00 0.00 1 0.000 0 0 2024-04-22T22:19:39.219 INFO:journalctl@ceph.mon.b.smithi086.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-22T22:19:39.219 INFO:journalctl@ceph.mon.b.smithi086.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-22T22:19:39.219 INFO:journalctl@ceph.mon.b.smithi086.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-22T22:19:39.220 INFO:journalctl@ceph.mon.b.smithi086.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-22T22:19:39.220 INFO:journalctl@ceph.mon.b.smithi086.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-22T22:19:39.220 INFO:journalctl@ceph.mon.b.smithi086.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-22T22:19:39.220 INFO:journalctl@ceph.mon.b.smithi086.stdout: Cumulative compaction: 0.00 GB write, 0.64 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T22:19:39.220 INFO:journalctl@ceph.mon.b.smithi086.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-22T22:19:39.220 INFO:journalctl@ceph.mon.b.smithi086.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-22T22:19:39.220 INFO:journalctl@ceph.mon.b.smithi086.stdout: 2024-04-22T22:19:39.220 INFO:journalctl@ceph.mon.b.smithi086.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-22T22:19:39.220 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: mon.b does not exist in monmap, will attempt to join an existing cluster 2024-04-22T22:19:39.220 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: using public_addr v2:172.21.15.86:0/0 -> [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] 2024-04-22T22:19:39.220 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: starting mon.b rank -1 at public addrs [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] at bind addrs [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 2024-04-22T22:19:39.220 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: mon.b@-1(???) e0 preinit fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 2024-04-22T22:19:39.220 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: mon.b@-1(synchronizing).mds e1 new map 2024-04-22T22:19:39.220 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: mon.b@-1(synchronizing).mds e1 print_map 2024-04-22T22:19:39.220 INFO:journalctl@ceph.mon.b.smithi086.stdout: e1 2024-04-22T22:19:39.220 INFO:journalctl@ceph.mon.b.smithi086.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-22T22:19:39.221 INFO:journalctl@ceph.mon.b.smithi086.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-22T22:19:39.221 INFO:journalctl@ceph.mon.b.smithi086.stdout: legacy client fscid: -1 2024-04-22T22:19:39.221 INFO:journalctl@ceph.mon.b.smithi086.stdout: 2024-04-22T22:19:39.221 INFO:journalctl@ceph.mon.b.smithi086.stdout: No filesystems configured 2024-04-22T22:19:39.221 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-04-22T22:19:39.221 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-04-22T22:19:39.221 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-04-22T22:19:39.221 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-04-22T22:19:39.221 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-04-22T22:19:39.221 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-04-22T22:19:39.221 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-22T22:19:39.221 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-22T22:19:39.221 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-22T22:19:39.221 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-22T22:19:39.221 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi067:172.21.15.67=a;smithi067:[v2:172.21.15.67:3301,v1:172.21.15.67:6790]=c;smithi086:172.21.15.86=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:19:39.221 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: Saving service mon spec with placement smithi067:172.21.15.67=a;smithi067:[v2:172.21.15.67:3301,v1:172.21.15.67:6790]=c;smithi086:172.21.15.86=b;count:3 2024-04-22T22:19:39.222 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: Updating smithi086:/etc/ceph/ceph.conf 2024-04-22T22:19:39.222 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: Updating smithi086:/etc/ceph/ceph.client.admin.keyring 2024-04-22T22:19:39.222 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:39.222 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:39.222 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T22:19:39.222 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:39.222 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: Deploying daemon mon.c on smithi067 2024-04-22T22:19:39.222 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:39.222 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: from='client.? 172.21.15.86:0/4038011321' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T22:19:39.222 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:39.222 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T22:19:39.222 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: mon.a calling monitor election 2024-04-22T22:19:39.222 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T22:19:39.222 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T22:19:39.222 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:39.223 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T22:19:39.223 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: mon.c calling monitor election 2024-04-22T22:19:39.223 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T22:19:39.223 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:39.223 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T22:19:39.223 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T22:19:39.223 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-22T22:19:39.223 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: monmap e2: 2 mons at {a=[v2:172.21.15.67:3300/0,v1:172.21.15.67:6789/0],c=[v2:172.21.15.67:3301/0,v1:172.21.15.67:6790/0]} 2024-04-22T22:19:39.223 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: fsmap 2024-04-22T22:19:39.223 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: osdmap e4: 0 total, 0 up, 0 in 2024-04-22T22:19:39.223 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: mgrmap e13: y(active, since 35s) 2024-04-22T22:19:39.223 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: overall HEALTH_OK 2024-04-22T22:19:39.223 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:39.223 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T22:19:39.223 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:39.223 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: Deploying daemon mon.b on smithi086 2024-04-22T22:19:39.224 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: from='client.? 172.21.15.86:0/3980945201' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T22:19:39.224 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T22:19:39.224 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:39.224 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:39.224 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-04-22T22:19:39.224 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 ceph-mon[22689]: mon.b@-1(synchronizing) e2 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-04-22T22:19:39.224 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 bash[22622]: be2e0891a57742b32181076130b29486a006138ea30755751c260c73b148f9f4 2024-04-22T22:19:39.224 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:38 smithi086 systemd[1]: Started Ceph mon.b for 1ab1a3a8-00f6-11ef-bc93-c7b262605968. 2024-04-22T22:19:43.865 INFO:teuthology.orchestra.run.smithi086.stdout: 2024-04-22T22:19:43.865 INFO:teuthology.orchestra.run.smithi086.stdout:{"epoch":3,"fsid":"1ab1a3a8-00f6-11ef-bc93-c7b262605968","modified":"2024-04-22T22:19:38.869914Z","created":"2024-04-22T22:18:06.454399Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:3300","nonce":0},{"type":"v1","addr":"172.21.15.67:6789","nonce":0}]},"addr":"172.21.15.67:6789/0","public_addr":"172.21.15.67:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:3301","nonce":0},{"type":"v1","addr":"172.21.15.67:6790","nonce":0}]},"addr":"172.21.15.67:6790/0","public_addr":"172.21.15.67:6790/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:3300","nonce":0},{"type":"v1","addr":"172.21.15.86:6789","nonce":0}]},"addr":"172.21.15.86:6789/0","public_addr":"172.21.15.86:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-04-22T22:19:43.868 INFO:teuthology.orchestra.run.smithi086.stderr:dumped monmap epoch 3 2024-04-22T22:19:44.146 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:43 smithi086 ceph-mon[22689]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:44.146 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:43 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T22:19:44.146 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:43 smithi086 ceph-mon[22689]: mon.a calling monitor election 2024-04-22T22:19:44.146 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:43 smithi086 ceph-mon[22689]: mon.c calling monitor election 2024-04-22T22:19:44.146 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:43 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T22:19:44.146 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:43 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T22:19:44.146 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:43 smithi086 ceph-mon[22689]: from='client.? 172.21.15.86:0/1363343800' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T22:19:44.146 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:43 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T22:19:44.147 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:43 smithi086 ceph-mon[22689]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:44.147 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:43 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T22:19:44.147 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:43 smithi086 ceph-mon[22689]: mon.b calling monitor election 2024-04-22T22:19:44.147 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:43 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T22:19:44.147 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:43 smithi086 ceph-mon[22689]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:44.147 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:43 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T22:19:44.147 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:43 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T22:19:44.147 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:43 smithi086 ceph-mon[22689]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-22T22:19:44.147 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:43 smithi086 ceph-mon[22689]: monmap e3: 3 mons at {a=[v2:172.21.15.67:3300/0,v1:172.21.15.67:6789/0],b=[v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0],c=[v2:172.21.15.67:3301/0,v1:172.21.15.67:6790/0]} 2024-04-22T22:19:44.147 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:43 smithi086 ceph-mon[22689]: fsmap 2024-04-22T22:19:44.147 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:43 smithi086 ceph-mon[22689]: osdmap e4: 0 total, 0 up, 0 in 2024-04-22T22:19:44.147 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:43 smithi086 ceph-mon[22689]: mgrmap e13: y(active, since 45s) 2024-04-22T22:19:44.147 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:43 smithi086 ceph-mon[22689]: overall HEALTH_OK 2024-04-22T22:19:44.147 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:43 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:44.147 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:43 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:19:44.147 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:43 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:44.147 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:43 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:19:44.249 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[21875]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:44.249 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T22:19:44.249 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[21875]: mon.a calling monitor election 2024-04-22T22:19:44.249 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[21875]: mon.c calling monitor election 2024-04-22T22:19:44.249 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T22:19:44.249 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T22:19:44.249 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[21875]: from='client.? 172.21.15.86:0/1363343800' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T22:19:44.249 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T22:19:44.249 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[21875]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:44.249 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T22:19:44.249 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[21875]: mon.b calling monitor election 2024-04-22T22:19:44.250 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T22:19:44.250 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[21875]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:44.250 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T22:19:44.250 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T22:19:44.250 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[21875]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-22T22:19:44.250 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[21875]: monmap e3: 3 mons at {a=[v2:172.21.15.67:3300/0,v1:172.21.15.67:6789/0],b=[v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0],c=[v2:172.21.15.67:3301/0,v1:172.21.15.67:6790/0]} 2024-04-22T22:19:44.250 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[21875]: fsmap 2024-04-22T22:19:44.250 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[21875]: osdmap e4: 0 total, 0 up, 0 in 2024-04-22T22:19:44.250 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[21875]: mgrmap e13: y(active, since 45s) 2024-04-22T22:19:44.250 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[21875]: overall HEALTH_OK 2024-04-22T22:19:44.250 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:44.250 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:19:44.250 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:44.250 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:19:44.251 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[26633]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:44.251 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T22:19:44.251 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[26633]: mon.a calling monitor election 2024-04-22T22:19:44.251 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[26633]: mon.c calling monitor election 2024-04-22T22:19:44.251 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T22:19:44.251 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T22:19:44.251 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[26633]: from='client.? 172.21.15.86:0/1363343800' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-22T22:19:44.251 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T22:19:44.251 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[26633]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:44.251 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T22:19:44.251 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[26633]: mon.b calling monitor election 2024-04-22T22:19:44.251 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T22:19:44.251 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[26633]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:44.252 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T22:19:44.252 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T22:19:44.252 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[26633]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-22T22:19:44.252 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[26633]: monmap e3: 3 mons at {a=[v2:172.21.15.67:3300/0,v1:172.21.15.67:6789/0],b=[v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0],c=[v2:172.21.15.67:3301/0,v1:172.21.15.67:6790/0]} 2024-04-22T22:19:44.252 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[26633]: fsmap 2024-04-22T22:19:44.252 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[26633]: osdmap e4: 0 total, 0 up, 0 in 2024-04-22T22:19:44.252 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[26633]: mgrmap e13: y(active, since 45s) 2024-04-22T22:19:44.252 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[26633]: overall HEALTH_OK 2024-04-22T22:19:44.252 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:44.252 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:19:44.252 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:44.253 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:43 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:19:44.477 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-04-22T22:19:44.477 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph config generate-minimal-conf 2024-04-22T22:19:45.617 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:45 smithi067 ceph-mon[21875]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:45.617 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:45 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T22:19:45.617 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:45 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:45.617 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:45 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:45.617 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:45 smithi067 ceph-mon[26633]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:45.617 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:45 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T22:19:45.617 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:45 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:45.617 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:45 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:45.997 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:45 smithi086 ceph-mon[22689]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:45.997 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:45 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T22:19:45.998 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:45 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:45.998 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:45 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:46.529 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:46 smithi067 ceph-mon[26633]: Updating smithi086:/etc/ceph/ceph.conf 2024-04-22T22:19:46.529 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:46 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:46.530 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:46 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:46.530 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:46 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:46.530 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:46 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T22:19:46.530 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:46 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T22:19:46.530 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:46 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:46.530 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:46 smithi067 ceph-mon[21875]: Updating smithi086:/etc/ceph/ceph.conf 2024-04-22T22:19:46.530 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:46 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:46.530 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:46 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:46.530 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:46 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:46.530 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:46 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T22:19:46.531 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:46 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T22:19:46.531 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:46 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:46.563 INFO:teuthology.orchestra.run.smithi067.stdout:# minimal ceph.conf for 1ab1a3a8-00f6-11ef-bc93-c7b262605968 2024-04-22T22:19:46.564 INFO:teuthology.orchestra.run.smithi067.stdout:[global] 2024-04-22T22:19:46.564 INFO:teuthology.orchestra.run.smithi067.stdout: fsid = 1ab1a3a8-00f6-11ef-bc93-c7b262605968 2024-04-22T22:19:46.564 INFO:teuthology.orchestra.run.smithi067.stdout: mon_host = [v2:172.21.15.67:3300/0,v1:172.21.15.67:6789/0] [v2:172.21.15.86:3300/0,v1:172.21.15.86:6789/0] [v2:172.21.15.67:3301/0,v1:172.21.15.67:6790/0] 2024-04-22T22:19:46.997 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:46 smithi086 ceph-mon[22689]: Updating smithi086:/etc/ceph/ceph.conf 2024-04-22T22:19:46.997 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:46 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:46.998 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:46 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:46.998 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:46 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:46.998 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:46 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T22:19:46.998 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:46 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T22:19:46.998 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:46 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:47.173 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-04-22T22:19:47.174 DEBUG:teuthology.orchestra.run.smithi067:> set -ex 2024-04-22T22:19:47.174 DEBUG:teuthology.orchestra.run.smithi067:> sudo dd of=/etc/ceph/ceph.conf 2024-04-22T22:19:47.200 DEBUG:teuthology.orchestra.run.smithi067:> set -ex 2024-04-22T22:19:47.200 DEBUG:teuthology.orchestra.run.smithi067:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-22T22:19:47.265 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-04-22T22:19:47.265 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd of=/etc/ceph/ceph.conf 2024-04-22T22:19:47.295 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-04-22T22:19:47.295 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-22T22:19:47.362 INFO:tasks.cephadm:Adding mgr.y on smithi067 2024-04-22T22:19:47.363 INFO:tasks.cephadm:Adding mgr.x on smithi086 2024-04-22T22:19:47.363 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph orch apply mgr '2;smithi067=y;smithi086=x' 2024-04-22T22:19:47.676 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:47 smithi086 ceph-mon[22689]: Updating smithi067:/etc/ceph/ceph.conf 2024-04-22T22:19:47.676 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:47 smithi086 ceph-mon[22689]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:47.676 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:47 smithi086 ceph-mon[22689]: Reconfiguring mon.a (unknown last config time)... 2024-04-22T22:19:47.676 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:47 smithi086 ceph-mon[22689]: Reconfiguring daemon mon.a on smithi067 2024-04-22T22:19:47.676 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:47 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/1968863443' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:47.749 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:47 smithi067 ceph-mon[21875]: Updating smithi067:/etc/ceph/ceph.conf 2024-04-22T22:19:47.749 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:47 smithi067 ceph-mon[21875]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:47.749 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:47 smithi067 ceph-mon[21875]: Reconfiguring mon.a (unknown last config time)... 2024-04-22T22:19:47.749 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:47 smithi067 ceph-mon[21875]: Reconfiguring daemon mon.a on smithi067 2024-04-22T22:19:47.749 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:47 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/1968863443' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:47.749 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:47 smithi067 ceph-mon[26633]: Updating smithi067:/etc/ceph/ceph.conf 2024-04-22T22:19:47.749 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:47 smithi067 ceph-mon[26633]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:47.750 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:47 smithi067 ceph-mon[26633]: Reconfiguring mon.a (unknown last config time)... 2024-04-22T22:19:47.750 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:47 smithi067 ceph-mon[26633]: Reconfiguring daemon mon.a on smithi067 2024-04-22T22:19:47.750 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:47 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/1968863443' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:49.056 INFO:teuthology.orchestra.run.smithi086.stdout:Scheduled mgr update... 2024-04-22T22:19:49.603 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:49 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:49.603 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:49 smithi067 ceph-mon[21875]: Reconfiguring mon.c (monmap changed)... 2024-04-22T22:19:49.603 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:49 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T22:19:49.603 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:49 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T22:19:49.603 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:49 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:49.603 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:49 smithi067 ceph-mon[21875]: Reconfiguring daemon mon.c on smithi067 2024-04-22T22:19:49.603 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:49 smithi067 ceph-mon[21875]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:49.603 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:49 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:49.604 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:49 smithi067 ceph-mon[21875]: from='client.24104 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi067=y;smithi086=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:19:49.604 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:49 smithi067 ceph-mon[21875]: Saving service mgr spec with placement smithi067=y;smithi086=x;count:2 2024-04-22T22:19:49.604 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:49 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:49.604 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:49 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:49.604 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:49 smithi067 ceph-mon[26633]: Reconfiguring mon.c (monmap changed)... 2024-04-22T22:19:49.604 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:49 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T22:19:49.604 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:49 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T22:19:49.604 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:49 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:49.604 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:49 smithi067 ceph-mon[26633]: Reconfiguring daemon mon.c on smithi067 2024-04-22T22:19:49.604 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:49 smithi067 ceph-mon[26633]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:49.604 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:49 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:49.604 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:49 smithi067 ceph-mon[26633]: from='client.24104 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi067=y;smithi086=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:19:49.604 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:49 smithi067 ceph-mon[26633]: Saving service mgr spec with placement smithi067=y;smithi086=x;count:2 2024-04-22T22:19:49.605 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:49 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:49.605 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:49 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:49.605 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:49 smithi086 ceph-mon[22689]: Reconfiguring mon.c (monmap changed)... 2024-04-22T22:19:49.605 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:49 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T22:19:49.605 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:49 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T22:19:49.605 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:49 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:49.605 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:49 smithi086 ceph-mon[22689]: Reconfiguring daemon mon.c on smithi067 2024-04-22T22:19:49.605 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:49 smithi086 ceph-mon[22689]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:49.605 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:49 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:49.605 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:49 smithi086 ceph-mon[22689]: from='client.24104 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi067=y;smithi086=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:19:49.606 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:49 smithi086 ceph-mon[22689]: Saving service mgr spec with placement smithi067=y;smithi086=x;count:2 2024-04-22T22:19:49.606 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:49 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:49.627 DEBUG:teuthology.orchestra.run.smithi086:mgr.x> sudo journalctl -f -n 0 -u ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@mgr.x.service 2024-04-22T22:19:49.629 INFO:tasks.cephadm:Deploying OSDs... 2024-04-22T22:19:49.630 DEBUG:teuthology.orchestra.run.smithi067:> set -ex 2024-04-22T22:19:49.630 DEBUG:teuthology.orchestra.run.smithi067:> dd if=/scratch_devs of=/dev/stdout 2024-04-22T22:19:49.649 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-22T22:19:49.649 DEBUG:teuthology.orchestra.run.smithi067:> stat /dev/vg_nvme/lv_1 2024-04-22T22:19:49.706 INFO:teuthology.orchestra.run.smithi067.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-22T22:19:49.706 INFO:teuthology.orchestra.run.smithi067.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T22:19:49.706 INFO:teuthology.orchestra.run.smithi067.stdout:Device: 5h/5d Inode: 667 Links: 1 2024-04-22T22:19:49.706 INFO:teuthology.orchestra.run.smithi067.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T22:19:49.706 INFO:teuthology.orchestra.run.smithi067.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T22:19:49.707 INFO:teuthology.orchestra.run.smithi067.stdout:Access: 2024-04-22 22:19:28.395728583 +0000 2024-04-22T22:19:49.707 INFO:teuthology.orchestra.run.smithi067.stdout:Modify: 2024-04-22 22:17:06.653729393 +0000 2024-04-22T22:19:49.707 INFO:teuthology.orchestra.run.smithi067.stdout:Change: 2024-04-22 22:17:06.653729393 +0000 2024-04-22T22:19:49.707 INFO:teuthology.orchestra.run.smithi067.stdout: Birth: 2024-04-22 22:17:06.653729393 +0000 2024-04-22T22:19:49.707 DEBUG:teuthology.orchestra.run.smithi067:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-22T22:19:49.773 INFO:teuthology.orchestra.run.smithi067.stderr:1+0 records in 2024-04-22T22:19:49.774 INFO:teuthology.orchestra.run.smithi067.stderr:1+0 records out 2024-04-22T22:19:49.774 INFO:teuthology.orchestra.run.smithi067.stderr:512 bytes copied, 0.000220877 s, 2.3 MB/s 2024-04-22T22:19:49.775 DEBUG:teuthology.orchestra.run.smithi067:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-22T22:19:49.834 DEBUG:teuthology.orchestra.run.smithi067:> stat /dev/vg_nvme/lv_2 2024-04-22T22:19:49.891 INFO:teuthology.orchestra.run.smithi067.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-22T22:19:49.891 INFO:teuthology.orchestra.run.smithi067.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T22:19:49.891 INFO:teuthology.orchestra.run.smithi067.stdout:Device: 5h/5d Inode: 685 Links: 1 2024-04-22T22:19:49.891 INFO:teuthology.orchestra.run.smithi067.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T22:19:49.891 INFO:teuthology.orchestra.run.smithi067.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T22:19:49.891 INFO:teuthology.orchestra.run.smithi067.stdout:Access: 2024-04-22 22:19:28.395728583 +0000 2024-04-22T22:19:49.891 INFO:teuthology.orchestra.run.smithi067.stdout:Modify: 2024-04-22 22:17:06.998721709 +0000 2024-04-22T22:19:49.891 INFO:teuthology.orchestra.run.smithi067.stdout:Change: 2024-04-22 22:17:06.998721709 +0000 2024-04-22T22:19:49.891 INFO:teuthology.orchestra.run.smithi067.stdout: Birth: 2024-04-22 22:17:06.998721709 +0000 2024-04-22T22:19:49.891 DEBUG:teuthology.orchestra.run.smithi067:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-22T22:19:49.956 INFO:teuthology.orchestra.run.smithi067.stderr:1+0 records in 2024-04-22T22:19:49.957 INFO:teuthology.orchestra.run.smithi067.stderr:1+0 records out 2024-04-22T22:19:49.957 INFO:teuthology.orchestra.run.smithi067.stderr:512 bytes copied, 0.000242753 s, 2.1 MB/s 2024-04-22T22:19:49.958 DEBUG:teuthology.orchestra.run.smithi067:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-22T22:19:50.016 DEBUG:teuthology.orchestra.run.smithi067:> stat /dev/vg_nvme/lv_3 2024-04-22T22:19:50.074 INFO:teuthology.orchestra.run.smithi067.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-22T22:19:50.074 INFO:teuthology.orchestra.run.smithi067.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T22:19:50.074 INFO:teuthology.orchestra.run.smithi067.stdout:Device: 5h/5d Inode: 703 Links: 1 2024-04-22T22:19:50.074 INFO:teuthology.orchestra.run.smithi067.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T22:19:50.074 INFO:teuthology.orchestra.run.smithi067.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T22:19:50.075 INFO:teuthology.orchestra.run.smithi067.stdout:Access: 2024-04-22 22:19:28.395728583 +0000 2024-04-22T22:19:50.075 INFO:teuthology.orchestra.run.smithi067.stdout:Modify: 2024-04-22 22:17:07.359713668 +0000 2024-04-22T22:19:50.075 INFO:teuthology.orchestra.run.smithi067.stdout:Change: 2024-04-22 22:17:07.359713668 +0000 2024-04-22T22:19:50.075 INFO:teuthology.orchestra.run.smithi067.stdout: Birth: 2024-04-22 22:17:07.359713668 +0000 2024-04-22T22:19:50.075 DEBUG:teuthology.orchestra.run.smithi067:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-22T22:19:50.140 INFO:teuthology.orchestra.run.smithi067.stderr:1+0 records in 2024-04-22T22:19:50.140 INFO:teuthology.orchestra.run.smithi067.stderr:1+0 records out 2024-04-22T22:19:50.140 INFO:teuthology.orchestra.run.smithi067.stderr:512 bytes copied, 0.000243215 s, 2.1 MB/s 2024-04-22T22:19:50.142 DEBUG:teuthology.orchestra.run.smithi067:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-22T22:19:50.199 DEBUG:teuthology.orchestra.run.smithi067:> stat /dev/vg_nvme/lv_4 2024-04-22T22:19:50.257 INFO:teuthology.orchestra.run.smithi067.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-22T22:19:50.257 INFO:teuthology.orchestra.run.smithi067.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T22:19:50.257 INFO:teuthology.orchestra.run.smithi067.stdout:Device: 5h/5d Inode: 721 Links: 1 2024-04-22T22:19:50.257 INFO:teuthology.orchestra.run.smithi067.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T22:19:50.257 INFO:teuthology.orchestra.run.smithi067.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T22:19:50.257 INFO:teuthology.orchestra.run.smithi067.stdout:Access: 2024-04-22 22:19:28.395728583 +0000 2024-04-22T22:19:50.257 INFO:teuthology.orchestra.run.smithi067.stdout:Modify: 2024-04-22 22:17:07.710705848 +0000 2024-04-22T22:19:50.257 INFO:teuthology.orchestra.run.smithi067.stdout:Change: 2024-04-22 22:17:07.710705848 +0000 2024-04-22T22:19:50.257 INFO:teuthology.orchestra.run.smithi067.stdout: Birth: 2024-04-22 22:17:07.710705848 +0000 2024-04-22T22:19:50.257 DEBUG:teuthology.orchestra.run.smithi067:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-22T22:19:50.321 INFO:teuthology.orchestra.run.smithi067.stderr:1+0 records in 2024-04-22T22:19:50.322 INFO:teuthology.orchestra.run.smithi067.stderr:1+0 records out 2024-04-22T22:19:50.322 INFO:teuthology.orchestra.run.smithi067.stderr:512 bytes copied, 0.000192806 s, 2.7 MB/s 2024-04-22T22:19:50.323 DEBUG:teuthology.orchestra.run.smithi067:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-22T22:19:50.381 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-04-22T22:19:50.381 DEBUG:teuthology.orchestra.run.smithi086:> dd if=/scratch_devs of=/dev/stdout 2024-04-22T22:19:50.396 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-22T22:19:50.397 DEBUG:teuthology.orchestra.run.smithi086:> stat /dev/vg_nvme/lv_1 2024-04-22T22:19:50.452 INFO:teuthology.orchestra.run.smithi086.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-22T22:19:50.452 INFO:teuthology.orchestra.run.smithi086.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T22:19:50.452 INFO:teuthology.orchestra.run.smithi086.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-04-22T22:19:50.452 INFO:teuthology.orchestra.run.smithi086.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T22:19:50.453 INFO:teuthology.orchestra.run.smithi086.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T22:19:50.453 INFO:teuthology.orchestra.run.smithi086.stdout:Access: 2024-04-22 22:19:37.553050982 +0000 2024-04-22T22:19:50.453 INFO:teuthology.orchestra.run.smithi086.stdout:Modify: 2024-04-22 22:17:08.522260623 +0000 2024-04-22T22:19:50.453 INFO:teuthology.orchestra.run.smithi086.stdout:Change: 2024-04-22 22:17:08.522260623 +0000 2024-04-22T22:19:50.453 INFO:teuthology.orchestra.run.smithi086.stdout: Birth: 2024-04-22 22:17:08.522260623 +0000 2024-04-22T22:19:50.453 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-22T22:19:50.515 INFO:teuthology.orchestra.run.smithi086.stderr:1+0 records in 2024-04-22T22:19:50.515 INFO:teuthology.orchestra.run.smithi086.stderr:1+0 records out 2024-04-22T22:19:50.515 INFO:teuthology.orchestra.run.smithi086.stderr:512 bytes copied, 0.000273147 s, 1.9 MB/s 2024-04-22T22:19:50.516 DEBUG:teuthology.orchestra.run.smithi086:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-22T22:19:50.571 DEBUG:teuthology.orchestra.run.smithi086:> stat /dev/vg_nvme/lv_2 2024-04-22T22:19:50.626 INFO:teuthology.orchestra.run.smithi086.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-22T22:19:50.626 INFO:teuthology.orchestra.run.smithi086.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T22:19:50.626 INFO:teuthology.orchestra.run.smithi086.stdout:Device: 5h/5d Inode: 685 Links: 1 2024-04-22T22:19:50.626 INFO:teuthology.orchestra.run.smithi086.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T22:19:50.626 INFO:teuthology.orchestra.run.smithi086.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T22:19:50.626 INFO:teuthology.orchestra.run.smithi086.stdout:Access: 2024-04-22 22:19:37.553050982 +0000 2024-04-22T22:19:50.626 INFO:teuthology.orchestra.run.smithi086.stdout:Modify: 2024-04-22 22:17:08.876253058 +0000 2024-04-22T22:19:50.626 INFO:teuthology.orchestra.run.smithi086.stdout:Change: 2024-04-22 22:17:08.876253058 +0000 2024-04-22T22:19:50.626 INFO:teuthology.orchestra.run.smithi086.stdout: Birth: 2024-04-22 22:17:08.876253058 +0000 2024-04-22T22:19:50.627 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-22T22:19:50.687 INFO:teuthology.orchestra.run.smithi086.stderr:1+0 records in 2024-04-22T22:19:50.687 INFO:teuthology.orchestra.run.smithi086.stderr:1+0 records out 2024-04-22T22:19:50.687 INFO:teuthology.orchestra.run.smithi086.stderr:512 bytes copied, 0.000219652 s, 2.3 MB/s 2024-04-22T22:19:50.689 DEBUG:teuthology.orchestra.run.smithi086:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-22T22:19:50.744 DEBUG:teuthology.orchestra.run.smithi086:> stat /dev/vg_nvme/lv_3 2024-04-22T22:19:50.799 INFO:teuthology.orchestra.run.smithi086.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-22T22:19:50.799 INFO:teuthology.orchestra.run.smithi086.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T22:19:50.799 INFO:teuthology.orchestra.run.smithi086.stdout:Device: 5h/5d Inode: 702 Links: 1 2024-04-22T22:19:50.799 INFO:teuthology.orchestra.run.smithi086.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T22:19:50.799 INFO:teuthology.orchestra.run.smithi086.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T22:19:50.799 INFO:teuthology.orchestra.run.smithi086.stdout:Access: 2024-04-22 22:19:37.554050961 +0000 2024-04-22T22:19:50.799 INFO:teuthology.orchestra.run.smithi086.stdout:Modify: 2024-04-22 22:17:09.224245620 +0000 2024-04-22T22:19:50.800 INFO:teuthology.orchestra.run.smithi086.stdout:Change: 2024-04-22 22:17:09.224245620 +0000 2024-04-22T22:19:50.800 INFO:teuthology.orchestra.run.smithi086.stdout: Birth: 2024-04-22 22:17:09.224245620 +0000 2024-04-22T22:19:50.800 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-22T22:19:50.861 INFO:teuthology.orchestra.run.smithi086.stderr:1+0 records in 2024-04-22T22:19:50.861 INFO:teuthology.orchestra.run.smithi086.stderr:1+0 records out 2024-04-22T22:19:50.861 INFO:teuthology.orchestra.run.smithi086.stderr:512 bytes copied, 0.000218358 s, 2.3 MB/s 2024-04-22T22:19:50.862 DEBUG:teuthology.orchestra.run.smithi086:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-22T22:19:50.917 DEBUG:teuthology.orchestra.run.smithi086:> stat /dev/vg_nvme/lv_4 2024-04-22T22:19:50.956 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:50 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:50.956 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:50 smithi086 ceph-mon[22689]: Reconfiguring mon.b (monmap changed)... 2024-04-22T22:19:50.957 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:50 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T22:19:50.957 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:50 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T22:19:50.957 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:50 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:50.957 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:50 smithi086 ceph-mon[22689]: Reconfiguring daemon mon.b on smithi086 2024-04-22T22:19:50.970 INFO:teuthology.orchestra.run.smithi086.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-22T22:19:50.970 INFO:teuthology.orchestra.run.smithi086.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-22T22:19:50.971 INFO:teuthology.orchestra.run.smithi086.stdout:Device: 5h/5d Inode: 724 Links: 1 2024-04-22T22:19:50.971 INFO:teuthology.orchestra.run.smithi086.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-22T22:19:50.971 INFO:teuthology.orchestra.run.smithi086.stdout:Context: system_u:object_r:device_t:s0 2024-04-22T22:19:50.971 INFO:teuthology.orchestra.run.smithi086.stdout:Access: 2024-04-22 22:19:37.554050961 +0000 2024-04-22T22:19:50.971 INFO:teuthology.orchestra.run.smithi086.stdout:Modify: 2024-04-22 22:17:09.575238119 +0000 2024-04-22T22:19:50.971 INFO:teuthology.orchestra.run.smithi086.stdout:Change: 2024-04-22 22:17:09.575238119 +0000 2024-04-22T22:19:50.971 INFO:teuthology.orchestra.run.smithi086.stdout: Birth: 2024-04-22 22:17:09.575238119 +0000 2024-04-22T22:19:50.972 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-22T22:19:50.999 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:50 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:50.999 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:50 smithi067 ceph-mon[21875]: Reconfiguring mon.b (monmap changed)... 2024-04-22T22:19:50.999 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:50 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T22:19:50.999 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:50 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T22:19:50.999 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:50 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:50.999 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:50 smithi067 ceph-mon[21875]: Reconfiguring daemon mon.b on smithi086 2024-04-22T22:19:50.999 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:50 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:51.000 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:50 smithi067 ceph-mon[26633]: Reconfiguring mon.b (monmap changed)... 2024-04-22T22:19:51.000 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:50 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-22T22:19:51.000 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:50 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-22T22:19:51.000 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:50 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:51.000 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:50 smithi067 ceph-mon[26633]: Reconfiguring daemon mon.b on smithi086 2024-04-22T22:19:51.033 INFO:teuthology.orchestra.run.smithi086.stderr:1+0 records in 2024-04-22T22:19:51.033 INFO:teuthology.orchestra.run.smithi086.stderr:1+0 records out 2024-04-22T22:19:51.033 INFO:teuthology.orchestra.run.smithi086.stderr:512 bytes copied, 0.000218654 s, 2.3 MB/s 2024-04-22T22:19:51.034 DEBUG:teuthology.orchestra.run.smithi086:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-22T22:19:51.090 INFO:tasks.cephadm:Deploying osd.0 on smithi067 with /dev/vg_nvme/lv_4... 2024-04-22T22:19:51.090 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-22T22:19:51.999 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:51 smithi067 ceph-mon[21875]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:51.999 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:51 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:51.999 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:51 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:19:51.999 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:51 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:52.000 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:51 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:19:52.000 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:51 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:52.000 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:51 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T22:19:52.000 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:51 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-22T22:19:52.000 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:51 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T22:19:52.000 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:51 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:52.000 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:51 smithi067 ceph-mon[21875]: Deploying daemon mgr.x on smithi086 2024-04-22T22:19:52.000 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:51 smithi067 ceph-mon[26633]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:52.000 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:51 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:52.000 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:51 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:19:52.000 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:51 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:52.000 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:51 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:19:52.001 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:51 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:52.001 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:51 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T22:19:52.001 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:51 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-22T22:19:52.001 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:51 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T22:19:52.001 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:51 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:52.001 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:51 smithi067 ceph-mon[26633]: Deploying daemon mgr.x on smithi086 2024-04-22T22:19:52.248 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:51 smithi086 ceph-mon[22689]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:52.248 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:51 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:52.248 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:51 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:19:52.248 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:51 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:52.248 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:51 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:19:52.248 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:51 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:52.248 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:51 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T22:19:52.248 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:51 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-22T22:19:52.248 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:51 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T22:19:52.248 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:51 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:52.248 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:51 smithi086 ceph-mon[22689]: Deploying daemon mgr.x on smithi086 2024-04-22T22:19:52.612 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:19:52 smithi086 systemd[1]: Starting Ceph mgr.x for 1ab1a3a8-00f6-11ef-bc93-c7b262605968... 2024-04-22T22:19:52.871 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:19:52 smithi086 podman[23745]: 2024-04-22 22:19:52.630525572 +0000 UTC m=+0.104841555 container create cd2bbd4855c30e295ff6e20cbc28be97add0724dbb4503ac4c9859f0f44861b8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, RELEASE=HEAD, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, name=centos-stream, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, architecture=x86_64) 2024-04-22T22:19:52.871 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:19:52 smithi086 podman[23745]: 2024-04-22 22:19:52.543834768 +0000 UTC m=+0.018150715 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-22T22:19:52.871 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:19:52 smithi086 podman[23745]: 2024-04-22 22:19:52.866333376 +0000 UTC m=+0.340649313 container init cd2bbd4855c30e295ff6e20cbc28be97add0724dbb4503ac4c9859f0f44861b8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, io.openshift.expose-services=, GIT_BRANCH=HEAD, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, maintainer=Guillaume Abrioux , vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-04-22T22:19:52.871 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:19:52 smithi086 podman[23745]: 2024-04-22 22:19:52.868966207 +0000 UTC m=+0.343282155 container start cd2bbd4855c30e295ff6e20cbc28be97add0724dbb4503ac4c9859f0f44861b8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, io.buildah.version=1.19.8, name=centos-stream, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, GIT_BRANCH=HEAD, vcs-type=git, release=754, architecture=x86_64, vendor=Red Hat, Inc., ceph=True, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux ) 2024-04-22T22:19:53.016 INFO:teuthology.orchestra.run.smithi067.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-22T22:19:53.016 INFO:teuthology.orchestra.run.smithi067.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-22T22:19:53.016 INFO:teuthology.orchestra.run.smithi067.stderr: stderr: 10+0 records in 2024-04-22T22:19:53.016 INFO:teuthology.orchestra.run.smithi067.stderr:10+0 records out 2024-04-22T22:19:53.017 INFO:teuthology.orchestra.run.smithi067.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0113745 s, 922 MB/s 2024-04-22T22:19:53.017 INFO:teuthology.orchestra.run.smithi067.stderr:--> Zapping successful for: 2024-04-22T22:19:53.127 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:19:52 smithi086 ceph-mgr[23764]: -- 172.21.15.86:0/2390723273 <== mon.2 v2:172.21.15.86:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x560c8cdf0340 con 0x560c8cdc8800 2024-04-22T22:19:53.127 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:19:52 smithi086 bash[23745]: cd2bbd4855c30e295ff6e20cbc28be97add0724dbb4503ac4c9859f0f44861b8 2024-04-22T22:19:53.128 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:19:52 smithi086 systemd[1]: Started Ceph mgr.x for 1ab1a3a8-00f6-11ef-bc93-c7b262605968. 2024-04-22T22:19:53.128 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:19:53 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:19:53.009+0000 7facf911d000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-22T22:19:53.128 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:19:53 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:19:53.145+0000 7facf911d000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-22T22:19:53.497 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:19:53 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:19:53.257+0000 7facf911d000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-22T22:19:53.624 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi067:vg_nvme/lv_4 2024-04-22T22:19:53.833 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:19:53 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:19:53.588+0000 7facf911d000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-22T22:19:53.833 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:19:53 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:19:53.672+0000 7facf911d000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-22T22:19:54.098 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:54 smithi086 ceph-mon[22689]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:54.098 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:54 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:54.098 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:54 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:19:54.098 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:54 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:54.098 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:54 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:19:54.098 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:54 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:54.249 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:54 smithi067 ceph-mon[26633]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:54.249 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:54 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:54.249 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:54 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:19:54.249 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:54 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:54.249 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:54 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:19:54.249 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:54 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:54.250 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:54 smithi067 ceph-mon[21875]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:54.250 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:54 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:54.250 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:54 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:19:54.250 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:54 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:54.250 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:54 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:19:54.250 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:54 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:55.104 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:19:54 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:19:54.811+0000 7facf911d000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-22T22:19:55.104 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:19:54 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:19:54.974+0000 7facf911d000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-22T22:19:55.104 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:19:55 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:19:55.054+0000 7facf911d000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-22T22:19:55.303 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:55 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:55.303 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:55 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:55.303 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:55 smithi067 ceph-mon[21875]: Reconfiguring mgr.y (unknown last config time)... 2024-04-22T22:19:55.303 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:55 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T22:19:55.303 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:55 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T22:19:55.303 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:55 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:55.303 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:55 smithi067 ceph-mon[21875]: Reconfiguring daemon mgr.y on smithi067 2024-04-22T22:19:55.303 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:55 smithi067 ceph-mon[21875]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:55.304 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:55 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:55.304 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:55 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:55.304 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:55 smithi067 ceph-mon[26633]: Reconfiguring mgr.y (unknown last config time)... 2024-04-22T22:19:55.304 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:55 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T22:19:55.304 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:55 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T22:19:55.304 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:55 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:55.304 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:55 smithi067 ceph-mon[26633]: Reconfiguring daemon mgr.y on smithi067 2024-04-22T22:19:55.304 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:55 smithi067 ceph-mon[26633]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:55.430 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:19:55 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:19:55.358+0000 7facf911d000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-22T22:19:55.430 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:55 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:55.430 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:55 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:55.430 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:55 smithi086 ceph-mon[22689]: Reconfiguring mgr.y (unknown last config time)... 2024-04-22T22:19:55.430 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:55 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T22:19:55.430 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:55 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T22:19:55.430 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:55 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:55.430 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:55 smithi086 ceph-mon[22689]: Reconfiguring daemon mgr.y on smithi067 2024-04-22T22:19:55.430 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:55 smithi086 ceph-mon[22689]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:55.748 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:19:55 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:19:55.448+0000 7facf911d000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-22T22:19:55.748 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:19:55 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:19:55.538+0000 7facf911d000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-22T22:19:56.110 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:19:55 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:19:55.772+0000 7facf911d000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-22T22:19:56.346 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:56 smithi067 ceph-mon[21875]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi067:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:19:56.347 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:56 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T22:19:56.347 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:56 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T22:19:56.347 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:56 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:56.347 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:56 smithi067 ceph-mon[26633]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi067:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:19:56.347 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:56 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T22:19:56.347 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:56 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T22:19:56.347 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:56 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:56.498 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:19:56 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:19:56.198+0000 7facf911d000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-22T22:19:56.498 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:56 smithi086 ceph-mon[22689]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi067:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:19:56.498 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:56 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T22:19:56.498 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:56 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T22:19:56.498 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:56 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:56.998 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:19:56 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:19:56.670+0000 7facf911d000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-22T22:19:56.998 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:19:56 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:19:56.760+0000 7facf911d000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-22T22:19:57.395 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:19:57 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:19:57.045+0000 7facf911d000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-22T22:19:57.395 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:19:57 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:19:57.130+0000 7facf911d000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-22T22:19:57.698 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:57 smithi067 ceph-mon[21875]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:57.698 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:57 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:57.698 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:57 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:19:57.698 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:57 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:57.698 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:57 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:19:57.698 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:57 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:57.698 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:57 smithi067 ceph-mon[26633]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:57.698 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:57 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:57.698 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:57 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:19:57.698 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:57 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:57.698 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:57 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:19:57.699 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:57 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:57.720 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:19:57 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:19:57.522+0000 7facf911d000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-22T22:19:57.721 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:19:57 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:19:57.649+0000 7facf911d000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-22T22:19:57.721 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:19:57 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:19:57.738+0000 7facf911d000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-22T22:19:57.721 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:57 smithi086 ceph-mon[22689]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:57.721 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:57 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:57.721 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:57 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:19:57.721 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:57 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:19:57.721 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:57 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:19:57.721 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:57 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:19:57.998 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:19:57 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:19:57.849+0000 7facf911d000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-22T22:19:57.998 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:19:57 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:19:57.934+0000 7facf911d000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-22T22:19:58.748 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:19:58 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:19:58.468+0000 7facf911d000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-22T22:19:58.748 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:58 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/1739155982' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "326d6f8e-d83f-4d1d-954d-0ee4e303d013"}]: dispatch 2024-04-22T22:19:58.748 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:58 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/1739155982' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "326d6f8e-d83f-4d1d-954d-0ee4e303d013"}]': finished 2024-04-22T22:19:58.748 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:58 smithi086 ceph-mon[22689]: osdmap e5: 1 total, 0 up, 1 in 2024-04-22T22:19:58.748 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:58 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T22:19:58.749 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:58 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/1739155982' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "326d6f8e-d83f-4d1d-954d-0ee4e303d013"}]: dispatch 2024-04-22T22:19:58.749 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:58 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/1739155982' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "326d6f8e-d83f-4d1d-954d-0ee4e303d013"}]': finished 2024-04-22T22:19:58.749 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:58 smithi067 ceph-mon[21875]: osdmap e5: 1 total, 0 up, 1 in 2024-04-22T22:19:58.749 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:58 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T22:19:58.750 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:58 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/1739155982' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "326d6f8e-d83f-4d1d-954d-0ee4e303d013"}]: dispatch 2024-04-22T22:19:58.750 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:58 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/1739155982' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "326d6f8e-d83f-4d1d-954d-0ee4e303d013"}]': finished 2024-04-22T22:19:58.750 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:58 smithi067 ceph-mon[26633]: osdmap e5: 1 total, 0 up, 1 in 2024-04-22T22:19:58.750 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:58 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T22:19:59.248 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:19:58 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:19:58.819+0000 7facf911d000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-22T22:19:59.748 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:59 smithi086 ceph-mon[22689]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:59.748 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:59 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:19:59.748 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:59 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:19:59.748 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:59 smithi086 ceph-mon[22689]: Standby manager daemon x started 2024-04-22T22:19:59.748 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:59 smithi086 ceph-mon[22689]: from='mgr.? 172.21.15.86:0/994539923' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T22:19:59.748 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:59 smithi086 ceph-mon[22689]: from='mgr.? 172.21.15.86:0/994539923' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T22:19:59.749 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:59 smithi086 ceph-mon[22689]: from='mgr.? 172.21.15.86:0/994539923' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T22:19:59.749 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:59 smithi086 ceph-mon[22689]: from='mgr.? 172.21.15.86:0/994539923' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T22:19:59.749 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:19:59 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/2457316024' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T22:19:59.749 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:59 smithi067 ceph-mon[21875]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:59.749 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:59 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:19:59.749 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:59 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:19:59.749 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:59 smithi067 ceph-mon[21875]: Standby manager daemon x started 2024-04-22T22:19:59.750 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:59 smithi067 ceph-mon[21875]: from='mgr.? 172.21.15.86:0/994539923' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T22:19:59.750 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:59 smithi067 ceph-mon[21875]: from='mgr.? 172.21.15.86:0/994539923' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T22:19:59.750 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:59 smithi067 ceph-mon[21875]: from='mgr.? 172.21.15.86:0/994539923' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T22:19:59.750 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:59 smithi067 ceph-mon[21875]: from='mgr.? 172.21.15.86:0/994539923' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T22:19:59.750 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:19:59 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/2457316024' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T22:19:59.750 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:59 smithi067 ceph-mon[26633]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:19:59.750 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:59 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:19:59.750 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:59 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:19:59.750 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:59 smithi067 ceph-mon[26633]: Standby manager daemon x started 2024-04-22T22:19:59.750 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:59 smithi067 ceph-mon[26633]: from='mgr.? 172.21.15.86:0/994539923' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T22:19:59.750 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:59 smithi067 ceph-mon[26633]: from='mgr.? 172.21.15.86:0/994539923' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T22:19:59.751 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:59 smithi067 ceph-mon[26633]: from='mgr.? 172.21.15.86:0/994539923' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T22:19:59.751 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:59 smithi067 ceph-mon[26633]: from='mgr.? 172.21.15.86:0/994539923' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T22:19:59.751 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:19:59 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/2457316024' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T22:20:00.748 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:00 smithi086 ceph-mon[22689]: mgrmap e14: y(active, since 61s), standbys: x 2024-04-22T22:20:00.748 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:00 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T22:20:00.748 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:00 smithi086 ceph-mon[22689]: overall HEALTH_OK 2024-04-22T22:20:00.749 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:00 smithi067 ceph-mon[21875]: mgrmap e14: y(active, since 61s), standbys: x 2024-04-22T22:20:00.749 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:00 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T22:20:00.749 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:00 smithi067 ceph-mon[21875]: overall HEALTH_OK 2024-04-22T22:20:00.750 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:00 smithi067 ceph-mon[26633]: mgrmap e14: y(active, since 61s), standbys: x 2024-04-22T22:20:00.750 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:00 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T22:20:00.750 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:00 smithi067 ceph-mon[26633]: overall HEALTH_OK 2024-04-22T22:20:01.735 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:01 smithi067 ceph-mon[21875]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:20:01.736 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:01 smithi067 ceph-mon[26633]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:20:01.998 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:01 smithi086 ceph-mon[22689]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:20:03.898 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:03 smithi067 ceph-mon[21875]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:20:03.898 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:03 smithi067 ceph-mon[26633]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:20:03.998 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:03 smithi086 ceph-mon[22689]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:20:04.800 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:04 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-22T22:20:04.800 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:04 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:20:04.801 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:04 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-22T22:20:04.801 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:04 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:20:04.998 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:04 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-22T22:20:04.998 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:04 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:20:05.740 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:05 smithi067 ceph-mon[21875]: Deploying daemon osd.0 on smithi067 2024-04-22T22:20:05.740 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:05 smithi067 ceph-mon[21875]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:20:05.741 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:05 smithi067 ceph-mon[26633]: Deploying daemon osd.0 on smithi067 2024-04-22T22:20:05.741 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:05 smithi067 ceph-mon[26633]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:20:05.998 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:05 smithi086 ceph-mon[22689]: Deploying daemon osd.0 on smithi067 2024-04-22T22:20:05.998 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:05 smithi086 ceph-mon[22689]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:20:07.750 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:07 smithi067 ceph-mon[21875]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:20:07.751 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:07 smithi067 ceph-mon[26633]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:20:07.998 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:07 smithi086 ceph-mon[22689]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:20:10.385 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:10 smithi067 ceph-mon[21875]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:20:10.385 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:10 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:10.385 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:10 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:20:10.385 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:10 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:20:10.386 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:10 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:20:10.386 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:10 smithi067 ceph-mon[26633]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:20:10.386 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:10 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:10.386 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:10 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:20:10.386 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:10 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:20:10.386 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:10 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:20:10.498 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:10 smithi086 ceph-mon[22689]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:20:10.499 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:10 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:10.499 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:10 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:20:10.499 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:10 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:20:10.499 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:10 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:20:11.944 INFO:teuthology.orchestra.run.smithi067.stdout:Created osd(s) 0 on host 'smithi067' 2024-04-22T22:20:12.174 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:12 smithi067 ceph-mon[21875]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:20:12.174 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:12 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:12.174 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:12 smithi067 ceph-mon[26633]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:20:12.174 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:12 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:12.498 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:12 smithi086 ceph-mon[22689]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:20:12.499 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:12 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:12.614 DEBUG:teuthology.orchestra.run.smithi067:osd.0> sudo journalctl -f -n 0 -u ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@osd.0.service 2024-04-22T22:20:12.617 INFO:tasks.cephadm:Deploying osd.1 on smithi067 with /dev/vg_nvme/lv_3... 2024-04-22T22:20:12.617 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-22T22:20:13.250 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:13 smithi067 ceph-mon[26633]: from='osd.0 [v2:172.21.15.67:6802/3032669792,v1:172.21.15.67:6803/3032669792]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-22T22:20:13.250 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:13 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:13.250 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:13 smithi067 ceph-mon[21875]: from='osd.0 [v2:172.21.15.67:6802/3032669792,v1:172.21.15.67:6803/3032669792]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-22T22:20:13.250 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:13 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:13.498 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:13 smithi086 ceph-mon[22689]: from='osd.0 [v2:172.21.15.67:6802/3032669792,v1:172.21.15.67:6803/3032669792]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-22T22:20:13.499 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:13 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:14.158 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:14 smithi067 ceph-mon[21875]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:20:14.158 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:14 smithi067 ceph-mon[21875]: from='osd.0 [v2:172.21.15.67:6802/3032669792,v1:172.21.15.67:6803/3032669792]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-22T22:20:14.158 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:14 smithi067 ceph-mon[21875]: osdmap e6: 1 total, 0 up, 1 in 2024-04-22T22:20:14.158 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:14 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T22:20:14.158 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:14 smithi067 ceph-mon[21875]: from='osd.0 [v2:172.21.15.67:6802/3032669792,v1:172.21.15.67:6803/3032669792]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi067", "root=default"]}]: dispatch 2024-04-22T22:20:14.158 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:14 smithi067 ceph-mon[26633]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:20:14.158 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:14 smithi067 ceph-mon[26633]: from='osd.0 [v2:172.21.15.67:6802/3032669792,v1:172.21.15.67:6803/3032669792]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-22T22:20:14.159 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:14 smithi067 ceph-mon[26633]: osdmap e6: 1 total, 0 up, 1 in 2024-04-22T22:20:14.159 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:14 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T22:20:14.159 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:14 smithi067 ceph-mon[26633]: from='osd.0 [v2:172.21.15.67:6802/3032669792,v1:172.21.15.67:6803/3032669792]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi067", "root=default"]}]: dispatch 2024-04-22T22:20:14.159 INFO:journalctl@ceph.osd.0.smithi067.stdout:Apr 22 22:20:14 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-0[29486]: 2024-04-22T22:20:14.092+0000 7f41637ce700 -1 osd.0 0 waiting for initial osdmap 2024-04-22T22:20:14.159 INFO:journalctl@ceph.osd.0.smithi067.stdout:Apr 22 22:20:14 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-0[29486]: 2024-04-22T22:20:14.095+0000 7f415c161700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T22:20:14.498 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:14 smithi086 ceph-mon[22689]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:20:14.499 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:14 smithi086 ceph-mon[22689]: from='osd.0 [v2:172.21.15.67:6802/3032669792,v1:172.21.15.67:6803/3032669792]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-22T22:20:14.499 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:14 smithi086 ceph-mon[22689]: osdmap e6: 1 total, 0 up, 1 in 2024-04-22T22:20:14.499 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:14 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T22:20:14.499 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:14 smithi086 ceph-mon[22689]: from='osd.0 [v2:172.21.15.67:6802/3032669792,v1:172.21.15.67:6803/3032669792]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi067", "root=default"]}]: dispatch 2024-04-22T22:20:15.193 INFO:teuthology.orchestra.run.smithi067.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-22T22:20:15.193 INFO:teuthology.orchestra.run.smithi067.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-22T22:20:15.193 INFO:teuthology.orchestra.run.smithi067.stderr: stderr: 10+0 records in 2024-04-22T22:20:15.193 INFO:teuthology.orchestra.run.smithi067.stderr:10+0 records out 2024-04-22T22:20:15.193 INFO:teuthology.orchestra.run.smithi067.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0125431 s, 836 MB/s 2024-04-22T22:20:15.193 INFO:teuthology.orchestra.run.smithi067.stderr:--> Zapping successful for: 2024-04-22T22:20:15.372 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:15 smithi067 ceph-mon[21875]: from='osd.0 [v2:172.21.15.67:6802/3032669792,v1:172.21.15.67:6803/3032669792]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi067", "root=default"]}]': finished 2024-04-22T22:20:15.372 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:15 smithi067 ceph-mon[21875]: osdmap e7: 1 total, 0 up, 1 in 2024-04-22T22:20:15.372 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:15 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T22:20:15.372 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:15 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T22:20:15.372 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:15 smithi067 ceph-mon[21875]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:20:15.372 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:15 smithi067 ceph-mon[26633]: from='osd.0 [v2:172.21.15.67:6802/3032669792,v1:172.21.15.67:6803/3032669792]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi067", "root=default"]}]': finished 2024-04-22T22:20:15.372 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:15 smithi067 ceph-mon[26633]: osdmap e7: 1 total, 0 up, 1 in 2024-04-22T22:20:15.373 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:15 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T22:20:15.373 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:15 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T22:20:15.373 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:15 smithi067 ceph-mon[26633]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:20:15.498 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:15 smithi086 ceph-mon[22689]: from='osd.0 [v2:172.21.15.67:6802/3032669792,v1:172.21.15.67:6803/3032669792]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi067", "root=default"]}]': finished 2024-04-22T22:20:15.499 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:15 smithi086 ceph-mon[22689]: osdmap e7: 1 total, 0 up, 1 in 2024-04-22T22:20:15.499 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:15 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T22:20:15.499 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:15 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T22:20:15.499 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:15 smithi086 ceph-mon[22689]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-22T22:20:15.986 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi067:vg_nvme/lv_3 2024-04-22T22:20:16.250 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:16 smithi067 ceph-mon[21875]: purged_snaps scrub starts 2024-04-22T22:20:16.250 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:16 smithi067 ceph-mon[21875]: purged_snaps scrub ok 2024-04-22T22:20:16.250 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:16 smithi067 ceph-mon[21875]: osd.0 [v2:172.21.15.67:6802/3032669792,v1:172.21.15.67:6803/3032669792] boot 2024-04-22T22:20:16.250 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:16 smithi067 ceph-mon[21875]: osdmap e8: 1 total, 1 up, 1 in 2024-04-22T22:20:16.250 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:16 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T22:20:16.251 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:16 smithi067 ceph-mon[26633]: purged_snaps scrub starts 2024-04-22T22:20:16.251 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:16 smithi067 ceph-mon[26633]: purged_snaps scrub ok 2024-04-22T22:20:16.251 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:16 smithi067 ceph-mon[26633]: osd.0 [v2:172.21.15.67:6802/3032669792,v1:172.21.15.67:6803/3032669792] boot 2024-04-22T22:20:16.251 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:16 smithi067 ceph-mon[26633]: osdmap e8: 1 total, 1 up, 1 in 2024-04-22T22:20:16.251 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:16 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T22:20:16.498 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:16 smithi086 ceph-mon[22689]: purged_snaps scrub starts 2024-04-22T22:20:16.499 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:16 smithi086 ceph-mon[22689]: purged_snaps scrub ok 2024-04-22T22:20:16.499 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:16 smithi086 ceph-mon[22689]: osd.0 [v2:172.21.15.67:6802/3032669792,v1:172.21.15.67:6803/3032669792] boot 2024-04-22T22:20:16.499 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:16 smithi086 ceph-mon[22689]: osdmap e8: 1 total, 1 up, 1 in 2024-04-22T22:20:16.499 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:16 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T22:20:17.250 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:17 smithi067 ceph-mon[21875]: osdmap e9: 1 total, 1 up, 1 in 2024-04-22T22:20:17.250 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:17 smithi067 ceph-mon[21875]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T22:20:17.250 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:17 smithi067 ceph-mon[26633]: osdmap e9: 1 total, 1 up, 1 in 2024-04-22T22:20:17.250 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:17 smithi067 ceph-mon[26633]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T22:20:17.498 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:17 smithi086 ceph-mon[22689]: osdmap e9: 1 total, 1 up, 1 in 2024-04-22T22:20:17.499 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:17 smithi086 ceph-mon[22689]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T22:20:18.749 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:18 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T22:20:18.749 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:18 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T22:20:18.749 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:18 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:20:18.750 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:18 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T22:20:18.750 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:18 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T22:20:18.750 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:18 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:20:18.750 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:18 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T22:20:18.751 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:18 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T22:20:18.751 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:18 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:20:19.874 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:19 smithi067 ceph-mon[21875]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T22:20:19.875 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:19 smithi067 ceph-mon[21875]: from='client.24145 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi067:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:20:19.875 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:19 smithi067 ceph-mon[21875]: Detected new or changed devices on smithi067 2024-04-22T22:20:19.875 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:19 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:19.875 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:19 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:20:19.875 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:19 smithi067 ceph-mon[21875]: Adjusting osd_memory_target on smithi067 to 16064M 2024-04-22T22:20:19.876 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:19 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:19.876 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:19 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:19.876 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:19 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:20:19.876 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:19 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:20:19.876 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:19 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:20:19.876 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:19 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:19.877 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:19 smithi067 ceph-mon[26633]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T22:20:19.877 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:19 smithi067 ceph-mon[26633]: from='client.24145 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi067:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:20:19.877 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:19 smithi067 ceph-mon[26633]: Detected new or changed devices on smithi067 2024-04-22T22:20:19.877 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:19 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:19.877 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:19 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:20:19.877 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:19 smithi067 ceph-mon[26633]: Adjusting osd_memory_target on smithi067 to 16064M 2024-04-22T22:20:19.878 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:19 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:19.878 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:19 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:19.878 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:19 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:20:19.878 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:19 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:20:19.878 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:19 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:20:19.878 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:19 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:19.999 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:19 smithi086 ceph-mon[22689]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T22:20:19.999 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:19 smithi086 ceph-mon[22689]: from='client.24145 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi067:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:20:19.999 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:19 smithi086 ceph-mon[22689]: Detected new or changed devices on smithi067 2024-04-22T22:20:19.999 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:19 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:19.999 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:19 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:20:19.999 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:19 smithi086 ceph-mon[22689]: Adjusting osd_memory_target on smithi067 to 16064M 2024-04-22T22:20:19.999 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:19 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:19.999 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:19 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:20.000 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:19 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:20:20.000 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:19 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:20:20.000 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:19 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:20:20.000 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:19 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:20.999 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:20 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/3141743768' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6d65e808-a2ac-4324-add8-bb0f331f6c0d"}]: dispatch 2024-04-22T22:20:20.999 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:20 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/3141743768' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6d65e808-a2ac-4324-add8-bb0f331f6c0d"}]': finished 2024-04-22T22:20:20.999 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:20 smithi086 ceph-mon[22689]: osdmap e10: 2 total, 1 up, 2 in 2024-04-22T22:20:20.999 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:20 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T22:20:21.000 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:20 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/3141743768' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6d65e808-a2ac-4324-add8-bb0f331f6c0d"}]: dispatch 2024-04-22T22:20:21.000 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:20 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/3141743768' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6d65e808-a2ac-4324-add8-bb0f331f6c0d"}]': finished 2024-04-22T22:20:21.000 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:20 smithi067 ceph-mon[21875]: osdmap e10: 2 total, 1 up, 2 in 2024-04-22T22:20:21.000 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:20 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T22:20:21.001 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:20 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/3141743768' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6d65e808-a2ac-4324-add8-bb0f331f6c0d"}]: dispatch 2024-04-22T22:20:21.001 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:20 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/3141743768' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6d65e808-a2ac-4324-add8-bb0f331f6c0d"}]': finished 2024-04-22T22:20:21.001 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:20 smithi067 ceph-mon[26633]: osdmap e10: 2 total, 1 up, 2 in 2024-04-22T22:20:21.001 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:20 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T22:20:21.999 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:21 smithi086 ceph-mon[22689]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T22:20:21.999 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:21 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/4060634584' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T22:20:22.000 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:21 smithi067 ceph-mon[21875]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T22:20:22.000 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:21 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/4060634584' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T22:20:22.000 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:21 smithi067 ceph-mon[26633]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T22:20:22.000 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:21 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/4060634584' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T22:20:23.750 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:23 smithi067 ceph-mon[21875]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T22:20:23.750 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:23 smithi067 ceph-mon[26633]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T22:20:23.999 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:23 smithi086 ceph-mon[22689]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T22:20:25.999 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:25 smithi086 ceph-mon[22689]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T22:20:26.000 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:25 smithi067 ceph-mon[21875]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T22:20:26.001 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:25 smithi067 ceph-mon[26633]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T22:20:27.999 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:27 smithi086 ceph-mon[22689]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T22:20:27.999 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:27 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-22T22:20:27.999 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:27 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:20:27.999 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:27 smithi086 ceph-mon[22689]: Deploying daemon osd.1 on smithi067 2024-04-22T22:20:28.000 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:27 smithi067 ceph-mon[26633]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T22:20:28.001 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:27 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-22T22:20:28.001 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:27 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:20:28.001 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:27 smithi067 ceph-mon[26633]: Deploying daemon osd.1 on smithi067 2024-04-22T22:20:28.001 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:27 smithi067 ceph-mon[21875]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T22:20:28.001 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:27 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-22T22:20:28.001 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:27 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:20:28.001 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:27 smithi067 ceph-mon[21875]: Deploying daemon osd.1 on smithi067 2024-04-22T22:20:29.750 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:29 smithi067 ceph-mon[21875]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T22:20:29.751 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:29 smithi067 ceph-mon[26633]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T22:20:29.999 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:29 smithi086 ceph-mon[22689]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T22:20:31.999 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:31 smithi086 ceph-mon[22689]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T22:20:32.000 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:31 smithi067 ceph-mon[21875]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T22:20:32.000 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:31 smithi067 ceph-mon[26633]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T22:20:32.999 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:32 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:32.999 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:32 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:20:33.000 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:32 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:20:33.000 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:32 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:20:33.000 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:32 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:33.001 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:32 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:20:33.001 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:32 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:20:33.001 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:32 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:20:33.001 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:32 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:33.001 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:32 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:20:33.001 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:32 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:20:33.001 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:32 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:20:33.997 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:33 smithi067 ceph-mon[21875]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T22:20:33.998 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:33 smithi067 ceph-mon[26633]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T22:20:33.999 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:33 smithi086 ceph-mon[22689]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T22:20:35.168 INFO:teuthology.orchestra.run.smithi067.stdout:Created osd(s) 1 on host 'smithi067' 2024-04-22T22:20:35.749 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:35 smithi086 ceph-mon[22689]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T22:20:35.749 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:35 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:35.749 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:35 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:35.749 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:35 smithi086 ceph-mon[22689]: from='osd.1 [v2:172.21.15.67:6810/740753600,v1:172.21.15.67:6811/740753600]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-22T22:20:35.750 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:35 smithi086 ceph-mon[22689]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-22T22:20:35.750 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:35 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:35.750 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:35 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:20:35.750 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:35 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:20:35.750 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:35 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:20:35.751 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:35 smithi067 ceph-mon[21875]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T22:20:35.751 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:35 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:35.751 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:35 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:35.751 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:35 smithi067 ceph-mon[21875]: from='osd.1 [v2:172.21.15.67:6810/740753600,v1:172.21.15.67:6811/740753600]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-22T22:20:35.751 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:35 smithi067 ceph-mon[21875]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-22T22:20:35.751 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:35 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:35.751 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:35 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:20:35.751 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:35 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:20:35.751 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:35 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:20:35.751 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:35 smithi067 ceph-mon[26633]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T22:20:35.751 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:35 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:35.751 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:35 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:35.751 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:35 smithi067 ceph-mon[26633]: from='osd.1 [v2:172.21.15.67:6810/740753600,v1:172.21.15.67:6811/740753600]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-22T22:20:35.751 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:35 smithi067 ceph-mon[26633]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-22T22:20:35.752 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:35 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:35.752 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:35 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:20:35.752 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:35 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:20:35.752 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:35 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:20:35.882 DEBUG:teuthology.orchestra.run.smithi067:osd.1> sudo journalctl -f -n 0 -u ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@osd.1.service 2024-04-22T22:20:35.884 INFO:tasks.cephadm:Deploying osd.2 on smithi067 with /dev/vg_nvme/lv_2... 2024-04-22T22:20:35.884 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-22T22:20:36.889 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:36 smithi067 ceph-mon[21875]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-22T22:20:36.889 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:36 smithi067 ceph-mon[21875]: osdmap e11: 2 total, 1 up, 2 in 2024-04-22T22:20:36.889 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:36 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T22:20:36.889 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:36 smithi067 ceph-mon[21875]: from='osd.1 [v2:172.21.15.67:6810/740753600,v1:172.21.15.67:6811/740753600]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi067", "root=default"]}]: dispatch 2024-04-22T22:20:36.890 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:36 smithi067 ceph-mon[21875]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi067", "root=default"]}]: dispatch 2024-04-22T22:20:36.890 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:36 smithi067 ceph-mon[26633]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-22T22:20:36.890 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:36 smithi067 ceph-mon[26633]: osdmap e11: 2 total, 1 up, 2 in 2024-04-22T22:20:36.890 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:36 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T22:20:36.890 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:36 smithi067 ceph-mon[26633]: from='osd.1 [v2:172.21.15.67:6810/740753600,v1:172.21.15.67:6811/740753600]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi067", "root=default"]}]: dispatch 2024-04-22T22:20:36.890 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:36 smithi067 ceph-mon[26633]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi067", "root=default"]}]: dispatch 2024-04-22T22:20:36.890 INFO:journalctl@ceph.osd.1.smithi067.stdout:Apr 22 22:20:36 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-1[32457]: 2024-04-22T22:20:36.773+0000 7fc0de3c2700 -1 osd.1 0 waiting for initial osdmap 2024-04-22T22:20:36.890 INFO:journalctl@ceph.osd.1.smithi067.stdout:Apr 22 22:20:36 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-1[32457]: 2024-04-22T22:20:36.776+0000 7fc0d6d55700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T22:20:37.249 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:36 smithi086 ceph-mon[22689]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-22T22:20:37.249 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:36 smithi086 ceph-mon[22689]: osdmap e11: 2 total, 1 up, 2 in 2024-04-22T22:20:37.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:36 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T22:20:37.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:36 smithi086 ceph-mon[22689]: from='osd.1 [v2:172.21.15.67:6810/740753600,v1:172.21.15.67:6811/740753600]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi067", "root=default"]}]: dispatch 2024-04-22T22:20:37.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:36 smithi086 ceph-mon[22689]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi067", "root=default"]}]: dispatch 2024-04-22T22:20:38.001 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:37 smithi067 ceph-mon[21875]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T22:20:38.001 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:37 smithi067 ceph-mon[21875]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi067", "root=default"]}]': finished 2024-04-22T22:20:38.002 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:37 smithi067 ceph-mon[21875]: osdmap e12: 2 total, 1 up, 2 in 2024-04-22T22:20:38.002 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:37 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T22:20:38.002 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:37 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T22:20:38.002 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:37 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T22:20:38.002 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:37 smithi067 ceph-mon[26633]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T22:20:38.002 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:37 smithi067 ceph-mon[26633]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi067", "root=default"]}]': finished 2024-04-22T22:20:38.002 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:37 smithi067 ceph-mon[26633]: osdmap e12: 2 total, 1 up, 2 in 2024-04-22T22:20:38.002 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:37 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T22:20:38.002 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:37 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T22:20:38.002 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:37 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T22:20:38.249 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:37 smithi086 ceph-mon[22689]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T22:20:38.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:37 smithi086 ceph-mon[22689]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi067", "root=default"]}]': finished 2024-04-22T22:20:38.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:37 smithi086 ceph-mon[22689]: osdmap e12: 2 total, 1 up, 2 in 2024-04-22T22:20:38.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:37 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T22:20:38.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:37 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T22:20:38.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:37 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T22:20:38.912 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:38 smithi067 ceph-mon[21875]: purged_snaps scrub starts 2024-04-22T22:20:38.912 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:38 smithi067 ceph-mon[21875]: purged_snaps scrub ok 2024-04-22T22:20:38.912 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:38 smithi067 ceph-mon[21875]: osd.1 [v2:172.21.15.67:6810/740753600,v1:172.21.15.67:6811/740753600] boot 2024-04-22T22:20:38.912 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:38 smithi067 ceph-mon[21875]: osdmap e13: 2 total, 2 up, 2 in 2024-04-22T22:20:38.912 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:38 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T22:20:38.913 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:38 smithi067 ceph-mon[26633]: purged_snaps scrub starts 2024-04-22T22:20:38.913 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:38 smithi067 ceph-mon[26633]: purged_snaps scrub ok 2024-04-22T22:20:38.913 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:38 smithi067 ceph-mon[26633]: osd.1 [v2:172.21.15.67:6810/740753600,v1:172.21.15.67:6811/740753600] boot 2024-04-22T22:20:38.913 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:38 smithi067 ceph-mon[26633]: osdmap e13: 2 total, 2 up, 2 in 2024-04-22T22:20:38.913 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:38 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T22:20:38.948 INFO:teuthology.orchestra.run.smithi067.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-22T22:20:38.948 INFO:teuthology.orchestra.run.smithi067.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-22T22:20:38.948 INFO:teuthology.orchestra.run.smithi067.stderr: stderr: 10+0 records in 2024-04-22T22:20:38.948 INFO:teuthology.orchestra.run.smithi067.stderr:10+0 records out 2024-04-22T22:20:38.948 INFO:teuthology.orchestra.run.smithi067.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0123539 s, 849 MB/s 2024-04-22T22:20:38.949 INFO:teuthology.orchestra.run.smithi067.stderr:--> Zapping successful for: 2024-04-22T22:20:39.249 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:38 smithi086 ceph-mon[22689]: purged_snaps scrub starts 2024-04-22T22:20:39.249 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:38 smithi086 ceph-mon[22689]: purged_snaps scrub ok 2024-04-22T22:20:39.249 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:38 smithi086 ceph-mon[22689]: osd.1 [v2:172.21.15.67:6810/740753600,v1:172.21.15.67:6811/740753600] boot 2024-04-22T22:20:39.249 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:38 smithi086 ceph-mon[22689]: osdmap e13: 2 total, 2 up, 2 in 2024-04-22T22:20:39.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:38 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T22:20:40.001 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:39 smithi067 ceph-mon[21875]: pgmap v53: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T22:20:40.001 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:39 smithi067 ceph-mon[21875]: osdmap e14: 2 total, 2 up, 2 in 2024-04-22T22:20:40.001 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:39 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:40.001 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:39 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:20:40.001 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:39 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:20:40.001 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:39 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:40.001 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:39 smithi067 ceph-mon[26633]: pgmap v53: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T22:20:40.002 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:39 smithi067 ceph-mon[26633]: osdmap e14: 2 total, 2 up, 2 in 2024-04-22T22:20:40.002 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:39 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:40.002 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:39 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:20:40.002 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:39 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:20:40.002 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:39 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:40.112 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi067:vg_nvme/lv_2 2024-04-22T22:20:40.151 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:39 smithi086 ceph-mon[22689]: pgmap v53: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-22T22:20:40.151 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:39 smithi086 ceph-mon[22689]: osdmap e14: 2 total, 2 up, 2 in 2024-04-22T22:20:40.151 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:39 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:40.151 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:39 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:20:40.151 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:39 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:20:40.151 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:39 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:40.999 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:40 smithi086 ceph-mon[22689]: Detected new or changed devices on smithi067 2024-04-22T22:20:41.000 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:40 smithi086 ceph-mon[22689]: Adjusting osd_memory_target on smithi067 to 8032M 2024-04-22T22:20:41.000 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:40 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:41.001 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:40 smithi067 ceph-mon[26633]: Detected new or changed devices on smithi067 2024-04-22T22:20:41.001 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:40 smithi067 ceph-mon[26633]: Adjusting osd_memory_target on smithi067 to 8032M 2024-04-22T22:20:41.001 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:40 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:41.001 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:40 smithi067 ceph-mon[21875]: Detected new or changed devices on smithi067 2024-04-22T22:20:41.001 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:40 smithi067 ceph-mon[21875]: Adjusting osd_memory_target on smithi067 to 8032M 2024-04-22T22:20:41.001 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:40 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:41.977 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:41 smithi067 ceph-mon[26633]: pgmap v55: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-22T22:20:41.977 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:41 smithi067 ceph-mon[21875]: pgmap v55: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-22T22:20:42.249 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:41 smithi086 ceph-mon[22689]: pgmap v55: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-22T22:20:43.001 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:42 smithi067 ceph-mon[21875]: from='client.24172 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi067:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:20:43.001 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:42 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T22:20:43.001 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:42 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T22:20:43.001 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:42 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:20:43.001 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:42 smithi067 ceph-mon[26633]: from='client.24172 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi067:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:20:43.001 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:42 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T22:20:43.002 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:42 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T22:20:43.002 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:42 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:20:43.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:42 smithi086 ceph-mon[22689]: from='client.24172 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi067:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:20:43.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:42 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T22:20:43.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:42 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T22:20:43.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:42 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:20:44.001 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:43 smithi067 ceph-mon[21875]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T22:20:44.001 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:43 smithi067 ceph-mon[26633]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T22:20:44.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:43 smithi086 ceph-mon[22689]: pgmap v56: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T22:20:45.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:44 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/4197639026' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c5c9cb79-a30b-4862-828b-7682f2ed658e"}]: dispatch 2024-04-22T22:20:45.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:44 smithi086 ceph-mon[22689]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c5c9cb79-a30b-4862-828b-7682f2ed658e"}]: dispatch 2024-04-22T22:20:45.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:44 smithi086 ceph-mon[22689]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c5c9cb79-a30b-4862-828b-7682f2ed658e"}]': finished 2024-04-22T22:20:45.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:44 smithi086 ceph-mon[22689]: osdmap e15: 3 total, 2 up, 3 in 2024-04-22T22:20:45.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:44 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T22:20:45.251 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:44 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/4197639026' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c5c9cb79-a30b-4862-828b-7682f2ed658e"}]: dispatch 2024-04-22T22:20:45.251 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:44 smithi067 ceph-mon[21875]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c5c9cb79-a30b-4862-828b-7682f2ed658e"}]: dispatch 2024-04-22T22:20:45.251 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:44 smithi067 ceph-mon[21875]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c5c9cb79-a30b-4862-828b-7682f2ed658e"}]': finished 2024-04-22T22:20:45.251 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:44 smithi067 ceph-mon[21875]: osdmap e15: 3 total, 2 up, 3 in 2024-04-22T22:20:45.251 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:44 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T22:20:45.251 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:44 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/4197639026' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c5c9cb79-a30b-4862-828b-7682f2ed658e"}]: dispatch 2024-04-22T22:20:45.251 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:44 smithi067 ceph-mon[26633]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c5c9cb79-a30b-4862-828b-7682f2ed658e"}]: dispatch 2024-04-22T22:20:45.252 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:44 smithi067 ceph-mon[26633]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c5c9cb79-a30b-4862-828b-7682f2ed658e"}]': finished 2024-04-22T22:20:45.252 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:44 smithi067 ceph-mon[26633]: osdmap e15: 3 total, 2 up, 3 in 2024-04-22T22:20:45.252 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:44 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T22:20:46.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:45 smithi086 ceph-mon[22689]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T22:20:46.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:45 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/4201363315' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T22:20:46.251 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:45 smithi067 ceph-mon[21875]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T22:20:46.251 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:45 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/4201363315' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T22:20:46.251 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:45 smithi067 ceph-mon[26633]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T22:20:46.251 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:45 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/4201363315' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T22:20:48.001 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:47 smithi067 ceph-mon[21875]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T22:20:48.002 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:47 smithi067 ceph-mon[26633]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T22:20:48.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:47 smithi086 ceph-mon[22689]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T22:20:50.055 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:49 smithi067 ceph-mon[21875]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T22:20:50.055 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:49 smithi067 ceph-mon[26633]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T22:20:50.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:49 smithi086 ceph-mon[22689]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T22:20:50.867 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:50 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-22T22:20:50.867 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:50 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:20:50.868 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:50 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-22T22:20:50.868 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:50 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:20:51.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:50 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-22T22:20:51.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:50 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:20:52.123 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:51 smithi067 ceph-mon[21875]: Deploying daemon osd.2 on smithi067 2024-04-22T22:20:52.123 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:51 smithi067 ceph-mon[21875]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T22:20:52.123 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:51 smithi067 ceph-mon[26633]: Deploying daemon osd.2 on smithi067 2024-04-22T22:20:52.123 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:51 smithi067 ceph-mon[26633]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T22:20:52.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:51 smithi086 ceph-mon[22689]: Deploying daemon osd.2 on smithi067 2024-04-22T22:20:52.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:51 smithi086 ceph-mon[22689]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T22:20:54.001 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:53 smithi067 ceph-mon[21875]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T22:20:54.002 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:53 smithi067 ceph-mon[26633]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T22:20:54.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:53 smithi086 ceph-mon[22689]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T22:20:55.952 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:55 smithi067 ceph-mon[26633]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T22:20:55.952 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:55 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:55.952 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:55 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:20:55.952 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:55 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:20:55.953 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:55 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:20:55.953 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:55 smithi067 ceph-mon[21875]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T22:20:55.953 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:55 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:55.953 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:55 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:20:55.953 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:55 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:20:55.953 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:55 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:20:56.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:55 smithi086 ceph-mon[22689]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T22:20:56.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:55 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:56.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:55 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:20:56.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:55 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:20:56.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:55 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:20:57.869 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:57 smithi067 ceph-mon[21875]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T22:20:57.869 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:57 smithi067 ceph-mon[26633]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T22:20:57.902 INFO:teuthology.orchestra.run.smithi067.stdout:Created osd(s) 2 on host 'smithi067' 2024-04-22T22:20:58.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:57 smithi086 ceph-mon[22689]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T22:20:58.834 DEBUG:teuthology.orchestra.run.smithi067:osd.2> sudo journalctl -f -n 0 -u ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@osd.2.service 2024-04-22T22:20:58.837 INFO:tasks.cephadm:Deploying osd.3 on smithi067 with /dev/vg_nvme/lv_1... 2024-04-22T22:20:58.837 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-22T22:20:59.078 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:58 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:59.078 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:58 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:20:59.078 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:58 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:20:59.078 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:58 smithi067 ceph-mon[21875]: from='osd.2 [v2:172.21.15.67:6818/1694104138,v1:172.21.15.67:6819/1694104138]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-22T22:20:59.078 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:58 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:59.078 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:58 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:20:59.078 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:58 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:20:59.078 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:58 smithi067 ceph-mon[26633]: from='osd.2 [v2:172.21.15.67:6818/1694104138,v1:172.21.15.67:6819/1694104138]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-22T22:20:59.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:58 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:20:59.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:58 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:20:59.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:58 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:20:59.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:58 smithi086 ceph-mon[22689]: from='osd.2 [v2:172.21.15.67:6818/1694104138,v1:172.21.15.67:6819/1694104138]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-22T22:21:00.081 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:59 smithi067 ceph-mon[21875]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T22:21:00.081 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:59 smithi067 ceph-mon[21875]: from='osd.2 [v2:172.21.15.67:6818/1694104138,v1:172.21.15.67:6819/1694104138]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-22T22:21:00.081 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:59 smithi067 ceph-mon[21875]: osdmap e16: 3 total, 2 up, 3 in 2024-04-22T22:21:00.081 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:59 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T22:21:00.081 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:59 smithi067 ceph-mon[21875]: from='osd.2 [v2:172.21.15.67:6818/1694104138,v1:172.21.15.67:6819/1694104138]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi067", "root=default"]}]: dispatch 2024-04-22T22:21:00.081 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:20:59 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:00.081 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:59 smithi067 ceph-mon[26633]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T22:21:00.081 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:59 smithi067 ceph-mon[26633]: from='osd.2 [v2:172.21.15.67:6818/1694104138,v1:172.21.15.67:6819/1694104138]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-22T22:21:00.081 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:59 smithi067 ceph-mon[26633]: osdmap e16: 3 total, 2 up, 3 in 2024-04-22T22:21:00.081 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:59 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T22:21:00.082 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:59 smithi067 ceph-mon[26633]: from='osd.2 [v2:172.21.15.67:6818/1694104138,v1:172.21.15.67:6819/1694104138]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi067", "root=default"]}]: dispatch 2024-04-22T22:21:00.082 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:20:59 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:00.082 INFO:journalctl@ceph.osd.2.smithi067.stdout:Apr 22 22:20:59 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-2[35377]: 2024-04-22T22:20:59.928+0000 7fee8aebf700 -1 osd.2 0 waiting for initial osdmap 2024-04-22T22:21:00.082 INFO:journalctl@ceph.osd.2.smithi067.stdout:Apr 22 22:20:59 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-2[35377]: 2024-04-22T22:20:59.931+0000 7fee85055700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T22:21:00.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:59 smithi086 ceph-mon[22689]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T22:21:00.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:59 smithi086 ceph-mon[22689]: from='osd.2 [v2:172.21.15.67:6818/1694104138,v1:172.21.15.67:6819/1694104138]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-22T22:21:00.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:59 smithi086 ceph-mon[22689]: osdmap e16: 3 total, 2 up, 3 in 2024-04-22T22:21:00.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:59 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T22:21:00.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:59 smithi086 ceph-mon[22689]: from='osd.2 [v2:172.21.15.67:6818/1694104138,v1:172.21.15.67:6819/1694104138]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi067", "root=default"]}]: dispatch 2024-04-22T22:21:00.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:20:59 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:01.096 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:00 smithi067 ceph-mon[21875]: from='osd.2 [v2:172.21.15.67:6818/1694104138,v1:172.21.15.67:6819/1694104138]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi067", "root=default"]}]': finished 2024-04-22T22:21:01.096 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:00 smithi067 ceph-mon[21875]: osdmap e17: 3 total, 2 up, 3 in 2024-04-22T22:21:01.096 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:00 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T22:21:01.097 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:00 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T22:21:01.097 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:00 smithi067 ceph-mon[26633]: from='osd.2 [v2:172.21.15.67:6818/1694104138,v1:172.21.15.67:6819/1694104138]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi067", "root=default"]}]': finished 2024-04-22T22:21:01.097 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:00 smithi067 ceph-mon[26633]: osdmap e17: 3 total, 2 up, 3 in 2024-04-22T22:21:01.097 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:00 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T22:21:01.097 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:00 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T22:21:01.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:00 smithi086 ceph-mon[22689]: from='osd.2 [v2:172.21.15.67:6818/1694104138,v1:172.21.15.67:6819/1694104138]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi067", "root=default"]}]': finished 2024-04-22T22:21:01.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:00 smithi086 ceph-mon[22689]: osdmap e17: 3 total, 2 up, 3 in 2024-04-22T22:21:01.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:00 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T22:21:01.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:00 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T22:21:01.491 INFO:teuthology.orchestra.run.smithi067.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-22T22:21:01.491 INFO:teuthology.orchestra.run.smithi067.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-22T22:21:01.491 INFO:teuthology.orchestra.run.smithi067.stderr: stderr: 10+0 records in 2024-04-22T22:21:01.491 INFO:teuthology.orchestra.run.smithi067.stderr:10+0 records out 2024-04-22T22:21:01.491 INFO:teuthology.orchestra.run.smithi067.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0141194 s, 743 MB/s 2024-04-22T22:21:01.492 INFO:teuthology.orchestra.run.smithi067.stderr:--> Zapping successful for: 2024-04-22T22:21:02.228 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:01 smithi067 ceph-mon[21875]: purged_snaps scrub starts 2024-04-22T22:21:02.228 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:01 smithi067 ceph-mon[21875]: purged_snaps scrub ok 2024-04-22T22:21:02.228 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:01 smithi067 ceph-mon[21875]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T22:21:02.228 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:01 smithi067 ceph-mon[21875]: osd.2 [v2:172.21.15.67:6818/1694104138,v1:172.21.15.67:6819/1694104138] boot 2024-04-22T22:21:02.228 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:01 smithi067 ceph-mon[21875]: osdmap e18: 3 total, 3 up, 3 in 2024-04-22T22:21:02.228 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:01 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T22:21:02.228 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:01 smithi067 ceph-mon[26633]: purged_snaps scrub starts 2024-04-22T22:21:02.229 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:01 smithi067 ceph-mon[26633]: purged_snaps scrub ok 2024-04-22T22:21:02.229 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:01 smithi067 ceph-mon[26633]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T22:21:02.229 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:01 smithi067 ceph-mon[26633]: osd.2 [v2:172.21.15.67:6818/1694104138,v1:172.21.15.67:6819/1694104138] boot 2024-04-22T22:21:02.229 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:01 smithi067 ceph-mon[26633]: osdmap e18: 3 total, 3 up, 3 in 2024-04-22T22:21:02.229 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:01 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T22:21:02.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:01 smithi086 ceph-mon[22689]: purged_snaps scrub starts 2024-04-22T22:21:02.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:01 smithi086 ceph-mon[22689]: purged_snaps scrub ok 2024-04-22T22:21:02.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:01 smithi086 ceph-mon[22689]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-22T22:21:02.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:01 smithi086 ceph-mon[22689]: osd.2 [v2:172.21.15.67:6818/1694104138,v1:172.21.15.67:6819/1694104138] boot 2024-04-22T22:21:02.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:01 smithi086 ceph-mon[22689]: osdmap e18: 3 total, 3 up, 3 in 2024-04-22T22:21:02.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:01 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T22:21:02.253 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi067:vg_nvme/lv_1 2024-04-22T22:21:03.176 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:02 smithi067 ceph-mon[26633]: osdmap e19: 3 total, 3 up, 3 in 2024-04-22T22:21:03.176 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:02 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-22T22:21:03.176 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:02 smithi067 ceph-mon[21875]: osdmap e19: 3 total, 3 up, 3 in 2024-04-22T22:21:03.176 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:02 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-22T22:21:03.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:02 smithi086 ceph-mon[22689]: osdmap e19: 3 total, 3 up, 3 in 2024-04-22T22:21:03.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:02 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-22T22:21:03.980 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:03 smithi067 ceph-mon[26633]: pgmap v71: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-22T22:21:03.980 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:03 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-22T22:21:03.980 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:03 smithi067 ceph-mon[26633]: osdmap e20: 3 total, 3 up, 3 in 2024-04-22T22:21:03.980 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:03 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-22T22:21:03.981 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:03 smithi067 ceph-mon[21875]: pgmap v71: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-22T22:21:03.981 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:03 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-22T22:21:03.981 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:03 smithi067 ceph-mon[21875]: osdmap e20: 3 total, 3 up, 3 in 2024-04-22T22:21:03.981 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:03 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-22T22:21:04.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:03 smithi086 ceph-mon[22689]: pgmap v71: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-22T22:21:04.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:03 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-22T22:21:04.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:03 smithi086 ceph-mon[22689]: osdmap e20: 3 total, 3 up, 3 in 2024-04-22T22:21:04.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:03 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-22T22:21:05.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:04 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-22T22:21:05.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:04 smithi086 ceph-mon[22689]: osdmap e21: 3 total, 3 up, 3 in 2024-04-22T22:21:05.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:04 smithi086 ceph-mon[22689]: Detected new or changed devices on smithi067 2024-04-22T22:21:05.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:04 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:05.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:04 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:21:05.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:04 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:21:05.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:04 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:21:05.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:04 smithi086 ceph-mon[22689]: Adjusting osd_memory_target on smithi067 to 5354M 2024-04-22T22:21:05.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:04 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:05.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:04 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:05.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:04 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:21:05.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:04 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:21:05.252 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:04 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:21:05.252 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:04 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:05.252 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:04 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-22T22:21:05.252 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:04 smithi067 ceph-mon[26633]: osdmap e21: 3 total, 3 up, 3 in 2024-04-22T22:21:05.252 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:04 smithi067 ceph-mon[26633]: Detected new or changed devices on smithi067 2024-04-22T22:21:05.252 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:04 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:05.252 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:04 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:21:05.252 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:04 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:21:05.252 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:04 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:21:05.253 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:04 smithi067 ceph-mon[26633]: Adjusting osd_memory_target on smithi067 to 5354M 2024-04-22T22:21:05.253 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:04 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:05.253 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:04 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:05.253 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:04 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:21:05.253 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:04 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:21:05.253 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:04 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:21:05.253 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:04 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:05.253 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:04 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-22T22:21:05.253 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:04 smithi067 ceph-mon[21875]: osdmap e21: 3 total, 3 up, 3 in 2024-04-22T22:21:05.253 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:04 smithi067 ceph-mon[21875]: Detected new or changed devices on smithi067 2024-04-22T22:21:05.253 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:04 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:05.253 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:04 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:21:05.253 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:04 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:21:05.254 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:04 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:21:05.254 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:04 smithi067 ceph-mon[21875]: Adjusting osd_memory_target on smithi067 to 5354M 2024-04-22T22:21:05.254 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:04 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:05.254 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:04 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:05.254 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:04 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:21:05.254 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:04 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:21:05.254 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:04 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:21:05.254 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:04 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:05.553 INFO:journalctl@ceph.osd.0.smithi067.stdout:Apr 22 22:21:05 smithi067 sudo[36763]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-22T22:21:05.553 INFO:journalctl@ceph.osd.0.smithi067.stdout:Apr 22 22:21:05 smithi067 sudo[36763]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T22:21:05.553 INFO:journalctl@ceph.osd.0.smithi067.stdout:Apr 22 22:21:05 smithi067 sudo[36763]: pam_unix(sudo:session): session closed for user root 2024-04-22T22:21:06.002 INFO:journalctl@ceph.osd.0.smithi067.stdout:Apr 22 22:21:05 smithi067 sudo[36823]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-22T22:21:06.002 INFO:journalctl@ceph.osd.0.smithi067.stdout:Apr 22 22:21:05 smithi067 sudo[36823]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T22:21:06.002 INFO:journalctl@ceph.osd.0.smithi067.stdout:Apr 22 22:21:05 smithi067 sudo[36823]: pam_unix(sudo:session): session closed for user root 2024-04-22T22:21:06.376 INFO:journalctl@ceph.osd.1.smithi067.stdout:Apr 22 22:21:06 smithi067 sudo[36829]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-22T22:21:06.376 INFO:journalctl@ceph.osd.1.smithi067.stdout:Apr 22 22:21:06 smithi067 sudo[36829]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T22:21:06.376 INFO:journalctl@ceph.osd.1.smithi067.stdout:Apr 22 22:21:06 smithi067 sudo[36829]: pam_unix(sudo:session): session closed for user root 2024-04-22T22:21:06.376 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:06 smithi067 ceph-mon[21875]: pgmap v74: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-22T22:21:06.376 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:06 smithi067 ceph-mon[21875]: from='client.14304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi067:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:21:06.376 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:06 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T22:21:06.377 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:06 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T22:21:06.377 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:06 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:21:06.377 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:06 smithi067 ceph-mon[21875]: osdmap e22: 3 total, 3 up, 3 in 2024-04-22T22:21:06.377 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:06 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:06.377 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:06 smithi067 ceph-mon[26633]: pgmap v74: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-22T22:21:06.377 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:06 smithi067 ceph-mon[26633]: from='client.14304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi067:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:21:06.377 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:06 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T22:21:06.377 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:06 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T22:21:06.377 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:06 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:21:06.377 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:06 smithi067 ceph-mon[26633]: osdmap e22: 3 total, 3 up, 3 in 2024-04-22T22:21:06.377 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:06 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:06.500 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:06 smithi086 ceph-mon[22689]: pgmap v74: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-22T22:21:06.500 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:06 smithi086 ceph-mon[22689]: from='client.14304 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi067:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:21:06.501 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:06 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T22:21:06.501 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:06 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T22:21:06.501 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:06 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:21:06.501 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:06 smithi086 ceph-mon[22689]: osdmap e22: 3 total, 3 up, 3 in 2024-04-22T22:21:06.501 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:06 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:07.002 INFO:journalctl@ceph.osd.1.smithi067.stdout:Apr 22 22:21:06 smithi067 sudo[36856]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-22T22:21:07.002 INFO:journalctl@ceph.osd.1.smithi067.stdout:Apr 22 22:21:06 smithi067 sudo[36856]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T22:21:07.002 INFO:journalctl@ceph.osd.1.smithi067.stdout:Apr 22 22:21:06 smithi067 sudo[36856]: pam_unix(sudo:session): session closed for user root 2024-04-22T22:21:07.502 INFO:journalctl@ceph.osd.2.smithi067.stdout:Apr 22 22:21:07 smithi067 sudo[36874]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-22T22:21:07.502 INFO:journalctl@ceph.osd.2.smithi067.stdout:Apr 22 22:21:07 smithi067 sudo[36874]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T22:21:07.502 INFO:journalctl@ceph.osd.2.smithi067.stdout:Apr 22 22:21:07 smithi067 sudo[36874]: pam_unix(sudo:session): session closed for user root 2024-04-22T22:21:08.001 INFO:journalctl@ceph.osd.2.smithi067.stdout:Apr 22 22:21:07 smithi067 sudo[36904]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-22T22:21:08.001 INFO:journalctl@ceph.osd.2.smithi067.stdout:Apr 22 22:21:07 smithi067 sudo[36904]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T22:21:08.001 INFO:journalctl@ceph.osd.2.smithi067.stdout:Apr 22 22:21:07 smithi067 sudo[36904]: pam_unix(sudo:session): session closed for user root 2024-04-22T22:21:08.252 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:08 smithi067 sudo[36945]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-22T22:21:08.252 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:08 smithi067 sudo[36945]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T22:21:08.252 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:08 smithi067 sudo[36945]: pam_unix(sudo:session): session closed for user root 2024-04-22T22:21:08.750 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:08 smithi086 ceph-mon[22689]: pgmap v76: 1 pgs: 1 creating+peering; 0 B data, 16 MiB used, 268 GiB / 268 GiB avail 2024-04-22T22:21:08.751 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:08 smithi086 ceph-mon[22689]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-22T22:21:08.751 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:08 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/170092339' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "90c05f07-2be9-45e6-b340-80d9fbbf6cc5"}]: dispatch 2024-04-22T22:21:08.751 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:08 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/170092339' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "90c05f07-2be9-45e6-b340-80d9fbbf6cc5"}]': finished 2024-04-22T22:21:08.751 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:08 smithi086 ceph-mon[22689]: osdmap e23: 4 total, 3 up, 4 in 2024-04-22T22:21:08.751 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:08 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T22:21:08.751 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:08 smithi067 sudo[36989]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-22T22:21:08.752 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:08 smithi067 sudo[36989]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T22:21:08.752 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:08 smithi067 sudo[36989]: pam_unix(sudo:session): session closed for user root 2024-04-22T22:21:08.752 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:08 smithi067 ceph-mon[21875]: pgmap v76: 1 pgs: 1 creating+peering; 0 B data, 16 MiB used, 268 GiB / 268 GiB avail 2024-04-22T22:21:08.752 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:08 smithi067 ceph-mon[21875]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-22T22:21:08.752 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:08 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/170092339' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "90c05f07-2be9-45e6-b340-80d9fbbf6cc5"}]: dispatch 2024-04-22T22:21:08.752 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:08 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/170092339' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "90c05f07-2be9-45e6-b340-80d9fbbf6cc5"}]': finished 2024-04-22T22:21:08.752 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:08 smithi067 ceph-mon[21875]: osdmap e23: 4 total, 3 up, 4 in 2024-04-22T22:21:08.752 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:08 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T22:21:09.252 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:08 smithi067 sudo[37363]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-22T22:21:09.252 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:08 smithi067 sudo[37363]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T22:21:09.252 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:08 smithi067 sudo[37363]: pam_unix(sudo:session): session closed for user root 2024-04-22T22:21:09.752 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:09 smithi067 sudo[37555]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-22T22:21:09.752 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:09 smithi067 sudo[37555]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T22:21:09.752 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:09 smithi067 sudo[37555]: pam_unix(sudo:session): session closed for user root 2024-04-22T22:21:09.752 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:09 smithi067 ceph-mon[26633]: pgmap v76: 1 pgs: 1 creating+peering; 0 B data, 16 MiB used, 268 GiB / 268 GiB avail 2024-04-22T22:21:09.752 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:09 smithi067 ceph-mon[26633]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-22T22:21:09.752 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:09 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/170092339' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "90c05f07-2be9-45e6-b340-80d9fbbf6cc5"}]: dispatch 2024-04-22T22:21:09.752 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:09 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/170092339' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "90c05f07-2be9-45e6-b340-80d9fbbf6cc5"}]': finished 2024-04-22T22:21:09.752 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:09 smithi067 ceph-mon[26633]: osdmap e23: 4 total, 3 up, 4 in 2024-04-22T22:21:09.752 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:09 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T22:21:09.853 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:09 smithi086 sudo[24096]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-22T22:21:09.853 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:09 smithi086 sudo[24096]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T22:21:09.853 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:09 smithi086 sudo[24096]: pam_unix(sudo:session): session closed for user root 2024-04-22T22:21:10.166 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:09 smithi067 ceph-mon[21875]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-22T22:21:10.166 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:09 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/2069049291' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T22:21:10.167 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:09 smithi067 ceph-mon[21875]: pgmap v78: 1 pgs: 1 creating+peering; 0 B data, 17 MiB used, 268 GiB / 268 GiB avail 2024-04-22T22:21:10.167 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:09 smithi067 ceph-mon[21875]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-22T22:21:10.167 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:09 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T22:21:10.167 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:09 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T22:21:10.167 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:09 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T22:21:10.167 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:09 smithi067 ceph-mon[21875]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-22T22:21:10.167 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:09 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T22:21:10.167 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:09 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T22:21:10.167 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:09 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T22:21:10.167 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:09 smithi067 ceph-mon[21875]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-22T22:21:10.167 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:09 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T22:21:10.167 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:09 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T22:21:10.167 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:09 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T22:21:10.167 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:09 smithi067 ceph-mon[26633]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-22T22:21:10.168 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:09 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/2069049291' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T22:21:10.168 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:09 smithi067 ceph-mon[26633]: pgmap v78: 1 pgs: 1 creating+peering; 0 B data, 17 MiB used, 268 GiB / 268 GiB avail 2024-04-22T22:21:10.168 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:09 smithi067 ceph-mon[26633]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-22T22:21:10.168 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:09 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T22:21:10.168 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:09 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T22:21:10.168 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:09 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T22:21:10.168 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:09 smithi067 ceph-mon[26633]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-22T22:21:10.168 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:09 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T22:21:10.168 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:09 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T22:21:10.168 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:09 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T22:21:10.168 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:09 smithi067 ceph-mon[26633]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-22T22:21:10.168 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:09 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T22:21:10.168 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:09 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T22:21:10.168 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:09 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T22:21:10.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:09 smithi086 sudo[24099]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-22T22:21:10.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:09 smithi086 sudo[24099]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-22T22:21:10.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:09 smithi086 sudo[24099]: pam_unix(sudo:session): session closed for user root 2024-04-22T22:21:10.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:09 smithi086 ceph-mon[22689]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-22T22:21:10.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:09 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/2069049291' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T22:21:10.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:09 smithi086 ceph-mon[22689]: pgmap v78: 1 pgs: 1 creating+peering; 0 B data, 17 MiB used, 268 GiB / 268 GiB avail 2024-04-22T22:21:10.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:09 smithi086 ceph-mon[22689]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-22T22:21:10.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:09 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T22:21:10.252 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:09 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T22:21:10.252 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:09 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T22:21:10.252 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:09 smithi086 ceph-mon[22689]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-22T22:21:10.252 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:09 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T22:21:10.252 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:09 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T22:21:10.252 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:09 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T22:21:10.252 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:09 smithi086 ceph-mon[22689]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-22T22:21:10.252 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:09 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T22:21:10.252 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:09 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T22:21:10.252 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:09 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T22:21:11.190 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:10 smithi067 ceph-mon[21875]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-22T22:21:11.190 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:10 smithi067 ceph-mon[21875]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-22T22:21:11.190 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:10 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:11.191 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:10 smithi067 ceph-mon[26633]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-22T22:21:11.191 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:10 smithi067 ceph-mon[26633]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-22T22:21:11.191 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:10 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:11.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:10 smithi086 ceph-mon[22689]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-22T22:21:11.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:10 smithi086 ceph-mon[22689]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-22T22:21:11.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:10 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:12.188 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:11 smithi067 ceph-mon[21875]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T22:21:12.188 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:11 smithi067 ceph-mon[26633]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T22:21:12.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:11 smithi086 ceph-mon[22689]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T22:21:14.002 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:13 smithi067 ceph-mon[21875]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T22:21:14.002 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:13 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-22T22:21:14.002 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:13 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:21:14.002 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:13 smithi067 ceph-mon[26633]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T22:21:14.003 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:13 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-22T22:21:14.003 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:13 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:21:14.250 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:13 smithi086 ceph-mon[22689]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T22:21:14.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:13 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-22T22:21:14.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:13 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:21:15.144 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:14 smithi067 ceph-mon[21875]: Deploying daemon osd.3 on smithi067 2024-04-22T22:21:15.144 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:14 smithi067 ceph-mon[26633]: Deploying daemon osd.3 on smithi067 2024-04-22T22:21:15.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:14 smithi086 ceph-mon[22689]: Deploying daemon osd.3 on smithi067 2024-04-22T22:21:16.030 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:15 smithi067 ceph-mon[26633]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-04-22T22:21:16.030 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:15 smithi067 ceph-mon[21875]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-04-22T22:21:16.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:15 smithi086 ceph-mon[22689]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-04-22T22:21:17.919 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:17 smithi067 ceph-mon[21875]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-04-22T22:21:17.919 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:17 smithi067 ceph-mon[26633]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-04-22T22:21:18.251 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:17 smithi086 ceph-mon[22689]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-04-22T22:21:19.737 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:19 smithi067 ceph-mon[21875]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-04-22T22:21:19.737 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:19 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:19.737 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:19 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:21:19.737 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:19 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:21:19.738 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:19 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:21:19.738 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:19 smithi067 ceph-mon[26633]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-04-22T22:21:19.738 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:19 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:19.738 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:19 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:21:19.738 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:19 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:21:19.738 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:19 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:21:19.751 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:19 smithi086 ceph-mon[22689]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-04-22T22:21:19.751 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:19 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:19.751 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:19 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:21:19.751 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:19 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:21:19.751 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:19 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:21:21.481 INFO:teuthology.orchestra.run.smithi067.stdout:Created osd(s) 3 on host 'smithi067' 2024-04-22T22:21:21.730 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:21 smithi067 ceph-mon[21875]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-22T22:21:21.731 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:21 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:21.731 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:21 smithi067 ceph-mon[26633]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-22T22:21:21.731 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:21 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:22.001 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:21 smithi086 ceph-mon[22689]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-22T22:21:22.001 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:21 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:22.114 DEBUG:teuthology.orchestra.run.smithi067:osd.3> sudo journalctl -f -n 0 -u ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@osd.3.service 2024-04-22T22:21:22.117 INFO:tasks.cephadm:Deploying osd.4 on smithi086 with /dev/vg_nvme/lv_4... 2024-04-22T22:21:22.117 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-22T22:21:22.632 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:22 smithi067 ceph-mon[26633]: from='osd.3 [v2:172.21.15.67:6826/2129594167,v1:172.21.15.67:6827/2129594167]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-22T22:21:22.632 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:22 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:22.632 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:22 smithi067 ceph-mon[21875]: from='osd.3 [v2:172.21.15.67:6826/2129594167,v1:172.21.15.67:6827/2129594167]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-22T22:21:22.633 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:22 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:22.727 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:22 smithi086 ceph-mon[22689]: from='osd.3 [v2:172.21.15.67:6826/2129594167,v1:172.21.15.67:6827/2129594167]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-22T22:21:22.727 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:22 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:23.565 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:23 smithi086 ceph-mon[22689]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-22T22:21:23.565 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:23 smithi086 ceph-mon[22689]: from='osd.3 [v2:172.21.15.67:6826/2129594167,v1:172.21.15.67:6827/2129594167]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-22T22:21:23.565 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:23 smithi086 ceph-mon[22689]: osdmap e24: 4 total, 3 up, 4 in 2024-04-22T22:21:23.565 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:23 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T22:21:23.565 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:23 smithi086 ceph-mon[22689]: from='osd.3 [v2:172.21.15.67:6826/2129594167,v1:172.21.15.67:6827/2129594167]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi067", "root=default"]}]: dispatch 2024-04-22T22:21:23.752 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:23 smithi067 ceph-mon[21875]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-22T22:21:23.753 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:23 smithi067 ceph-mon[21875]: from='osd.3 [v2:172.21.15.67:6826/2129594167,v1:172.21.15.67:6827/2129594167]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-22T22:21:23.753 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:23 smithi067 ceph-mon[21875]: osdmap e24: 4 total, 3 up, 4 in 2024-04-22T22:21:23.753 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:23 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T22:21:23.753 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:23 smithi067 ceph-mon[21875]: from='osd.3 [v2:172.21.15.67:6826/2129594167,v1:172.21.15.67:6827/2129594167]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi067", "root=default"]}]: dispatch 2024-04-22T22:21:23.753 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:23 smithi067 ceph-mon[26633]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-22T22:21:23.753 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:23 smithi067 ceph-mon[26633]: from='osd.3 [v2:172.21.15.67:6826/2129594167,v1:172.21.15.67:6827/2129594167]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-22T22:21:23.753 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:23 smithi067 ceph-mon[26633]: osdmap e24: 4 total, 3 up, 4 in 2024-04-22T22:21:23.753 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:23 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T22:21:23.753 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:23 smithi067 ceph-mon[26633]: from='osd.3 [v2:172.21.15.67:6826/2129594167,v1:172.21.15.67:6827/2129594167]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi067", "root=default"]}]: dispatch 2024-04-22T22:21:23.754 INFO:journalctl@ceph.osd.3.smithi067.stdout:Apr 22 22:21:23 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-3[38366]: 2024-04-22T22:21:23.554+0000 7fd8b4192700 -1 osd.3 0 waiting for initial osdmap 2024-04-22T22:21:23.754 INFO:journalctl@ceph.osd.3.smithi067.stdout:Apr 22 22:21:23 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-3[38366]: 2024-04-22T22:21:23.560+0000 7fd8af32a700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T22:21:24.065 INFO:teuthology.orchestra.run.smithi086.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-22T22:21:24.066 INFO:teuthology.orchestra.run.smithi086.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-22T22:21:24.066 INFO:teuthology.orchestra.run.smithi086.stderr: stderr: 10+0 records in 2024-04-22T22:21:24.066 INFO:teuthology.orchestra.run.smithi086.stderr:10+0 records out 2024-04-22T22:21:24.066 INFO:teuthology.orchestra.run.smithi086.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0113425 s, 924 MB/s 2024-04-22T22:21:24.066 INFO:teuthology.orchestra.run.smithi086.stderr:--> Zapping successful for: 2024-04-22T22:21:24.612 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:24 smithi086 ceph-mon[22689]: from='osd.3 [v2:172.21.15.67:6826/2129594167,v1:172.21.15.67:6827/2129594167]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi067", "root=default"]}]': finished 2024-04-22T22:21:24.612 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:24 smithi086 ceph-mon[22689]: osdmap e25: 4 total, 3 up, 4 in 2024-04-22T22:21:24.612 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:24 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T22:21:24.612 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:24 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T22:21:24.635 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi086:vg_nvme/lv_4 2024-04-22T22:21:25.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:24 smithi067 ceph-mon[21875]: from='osd.3 [v2:172.21.15.67:6826/2129594167,v1:172.21.15.67:6827/2129594167]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi067", "root=default"]}]': finished 2024-04-22T22:21:25.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:24 smithi067 ceph-mon[21875]: osdmap e25: 4 total, 3 up, 4 in 2024-04-22T22:21:25.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:24 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T22:21:25.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:24 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T22:21:25.003 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:24 smithi067 ceph-mon[26633]: from='osd.3 [v2:172.21.15.67:6826/2129594167,v1:172.21.15.67:6827/2129594167]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi067", "root=default"]}]': finished 2024-04-22T22:21:25.003 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:24 smithi067 ceph-mon[26633]: osdmap e25: 4 total, 3 up, 4 in 2024-04-22T22:21:25.003 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:24 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T22:21:25.003 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:24 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T22:21:25.585 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:25 smithi067 ceph-mon[26633]: purged_snaps scrub starts 2024-04-22T22:21:25.585 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:25 smithi067 ceph-mon[26633]: purged_snaps scrub ok 2024-04-22T22:21:25.585 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:25 smithi067 ceph-mon[26633]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T22:21:25.585 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:25 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T22:21:25.586 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:25 smithi067 ceph-mon[26633]: osd.3 [v2:172.21.15.67:6826/2129594167,v1:172.21.15.67:6827/2129594167] boot 2024-04-22T22:21:25.586 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:25 smithi067 ceph-mon[26633]: osdmap e26: 4 total, 4 up, 4 in 2024-04-22T22:21:25.586 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:25 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T22:21:25.586 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:25 smithi067 ceph-mon[21875]: purged_snaps scrub starts 2024-04-22T22:21:25.586 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:25 smithi067 ceph-mon[21875]: purged_snaps scrub ok 2024-04-22T22:21:25.586 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:25 smithi067 ceph-mon[21875]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T22:21:25.586 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:25 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T22:21:25.586 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:25 smithi067 ceph-mon[21875]: osd.3 [v2:172.21.15.67:6826/2129594167,v1:172.21.15.67:6827/2129594167] boot 2024-04-22T22:21:25.586 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:25 smithi067 ceph-mon[21875]: osdmap e26: 4 total, 4 up, 4 in 2024-04-22T22:21:25.586 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:25 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T22:21:25.851 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:25 smithi086 ceph-mon[22689]: purged_snaps scrub starts 2024-04-22T22:21:25.851 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:25 smithi086 ceph-mon[22689]: purged_snaps scrub ok 2024-04-22T22:21:25.851 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:25 smithi086 ceph-mon[22689]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-22T22:21:25.851 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:25 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T22:21:25.851 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:25 smithi086 ceph-mon[22689]: osd.3 [v2:172.21.15.67:6826/2129594167,v1:172.21.15.67:6827/2129594167] boot 2024-04-22T22:21:25.851 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:25 smithi086 ceph-mon[22689]: osdmap e26: 4 total, 4 up, 4 in 2024-04-22T22:21:25.851 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:25 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T22:21:26.751 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:26 smithi086 ceph-mon[22689]: osdmap e27: 4 total, 4 up, 4 in 2024-04-22T22:21:26.752 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:26 smithi086 ceph-mon[22689]: Detected new or changed devices on smithi067 2024-04-22T22:21:26.752 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:26 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:26.752 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:26 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:21:26.752 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:26 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:21:26.752 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:26 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:21:26.752 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:26 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:21:26.752 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:26 smithi086 ceph-mon[22689]: Adjusting osd_memory_target on smithi067 to 4016M 2024-04-22T22:21:26.752 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:26 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:26.752 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:26 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:26.752 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:26 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:21:26.752 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:26 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:21:26.752 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:26 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:21:26.752 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:26 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:26.753 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:26 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T22:21:26.753 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:26 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T22:21:26.753 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:26 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:21:27.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:26 smithi067 ceph-mon[21875]: osdmap e27: 4 total, 4 up, 4 in 2024-04-22T22:21:27.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:26 smithi067 ceph-mon[21875]: Detected new or changed devices on smithi067 2024-04-22T22:21:27.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:26 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:27.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:26 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:21:27.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:26 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:21:27.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:26 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:21:27.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:26 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:21:27.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:26 smithi067 ceph-mon[21875]: Adjusting osd_memory_target on smithi067 to 4016M 2024-04-22T22:21:27.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:26 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:27.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:26 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:27.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:26 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:21:27.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:26 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:21:27.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:26 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:21:27.004 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:26 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:27.004 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:26 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T22:21:27.004 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:26 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T22:21:27.004 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:26 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:21:27.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:26 smithi067 ceph-mon[26633]: osdmap e27: 4 total, 4 up, 4 in 2024-04-22T22:21:27.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:26 smithi067 ceph-mon[26633]: Detected new or changed devices on smithi067 2024-04-22T22:21:27.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:26 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:27.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:26 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:21:27.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:26 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:21:27.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:26 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:21:27.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:26 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:21:27.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:26 smithi067 ceph-mon[26633]: Adjusting osd_memory_target on smithi067 to 4016M 2024-04-22T22:21:27.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:26 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:27.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:26 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:27.005 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:26 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:21:27.005 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:26 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:21:27.005 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:26 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:21:27.005 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:26 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:27.005 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:26 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T22:21:27.005 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:26 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T22:21:27.005 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:26 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:21:27.819 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:27 smithi086 ceph-mon[22689]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T22:21:27.820 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:27 smithi086 ceph-mon[22689]: from='client.24212 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi086:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:21:28.002 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:27 smithi067 ceph-mon[21875]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T22:21:28.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:27 smithi067 ceph-mon[21875]: from='client.24212 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi086:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:21:28.003 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:27 smithi067 ceph-mon[26633]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T22:21:28.003 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:27 smithi067 ceph-mon[26633]: from='client.24212 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi086:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:21:29.001 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:28 smithi086 ceph-mon[22689]: from='client.? 172.21.15.86:0/2473781378' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a5d712df-0b70-465e-8462-0058860492a9"}]: dispatch 2024-04-22T22:21:29.002 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:28 smithi086 ceph-mon[22689]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a5d712df-0b70-465e-8462-0058860492a9"}]: dispatch 2024-04-22T22:21:29.002 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:28 smithi086 ceph-mon[22689]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a5d712df-0b70-465e-8462-0058860492a9"}]': finished 2024-04-22T22:21:29.002 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:28 smithi086 ceph-mon[22689]: osdmap e28: 5 total, 4 up, 5 in 2024-04-22T22:21:29.002 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:28 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T22:21:29.002 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:28 smithi067 ceph-mon[21875]: from='client.? 172.21.15.86:0/2473781378' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a5d712df-0b70-465e-8462-0058860492a9"}]: dispatch 2024-04-22T22:21:29.002 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:28 smithi067 ceph-mon[21875]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a5d712df-0b70-465e-8462-0058860492a9"}]: dispatch 2024-04-22T22:21:29.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:28 smithi067 ceph-mon[21875]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a5d712df-0b70-465e-8462-0058860492a9"}]': finished 2024-04-22T22:21:29.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:28 smithi067 ceph-mon[21875]: osdmap e28: 5 total, 4 up, 5 in 2024-04-22T22:21:29.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:28 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T22:21:29.003 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:28 smithi067 ceph-mon[26633]: from='client.? 172.21.15.86:0/2473781378' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a5d712df-0b70-465e-8462-0058860492a9"}]: dispatch 2024-04-22T22:21:29.003 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:28 smithi067 ceph-mon[26633]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a5d712df-0b70-465e-8462-0058860492a9"}]: dispatch 2024-04-22T22:21:29.003 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:28 smithi067 ceph-mon[26633]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a5d712df-0b70-465e-8462-0058860492a9"}]': finished 2024-04-22T22:21:29.003 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:28 smithi067 ceph-mon[26633]: osdmap e28: 5 total, 4 up, 5 in 2024-04-22T22:21:29.003 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:28 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T22:21:30.001 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:29 smithi086 ceph-mon[22689]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T22:21:30.001 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:29 smithi086 ceph-mon[22689]: from='client.? 172.21.15.86:0/607726002' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T22:21:30.002 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:29 smithi067 ceph-mon[21875]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T22:21:30.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:29 smithi067 ceph-mon[21875]: from='client.? 172.21.15.86:0/607726002' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T22:21:30.003 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:29 smithi067 ceph-mon[26633]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T22:21:30.003 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:29 smithi067 ceph-mon[26633]: from='client.? 172.21.15.86:0/607726002' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T22:21:31.985 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:31 smithi086 ceph-mon[22689]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T22:21:32.002 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:31 smithi067 ceph-mon[26633]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T22:21:32.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:31 smithi067 ceph-mon[21875]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T22:21:33.728 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:33 smithi086 ceph-mon[22689]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T22:21:34.003 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:33 smithi067 ceph-mon[26633]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T22:21:34.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:33 smithi067 ceph-mon[21875]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T22:21:35.751 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:35 smithi086 ceph-mon[22689]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T22:21:35.751 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:35 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-22T22:21:35.752 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:35 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:21:35.752 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:35 smithi086 ceph-mon[22689]: Deploying daemon osd.4 on smithi086 2024-04-22T22:21:36.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:35 smithi067 ceph-mon[21875]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T22:21:36.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:35 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-22T22:21:36.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:35 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:21:36.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:35 smithi067 ceph-mon[21875]: Deploying daemon osd.4 on smithi086 2024-04-22T22:21:36.003 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:35 smithi067 ceph-mon[26633]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T22:21:36.003 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:35 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-22T22:21:36.003 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:35 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:21:36.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:35 smithi067 ceph-mon[26633]: Deploying daemon osd.4 on smithi086 2024-04-22T22:21:37.746 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:37 smithi086 ceph-mon[22689]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T22:21:38.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:37 smithi067 ceph-mon[21875]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T22:21:38.003 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:37 smithi067 ceph-mon[26633]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T22:21:39.612 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:39 smithi086 ceph-mon[22689]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T22:21:40.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:39 smithi067 ceph-mon[21875]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T22:21:40.003 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:39 smithi067 ceph-mon[26633]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T22:21:40.935 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:40 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:40.935 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:40 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:21:40.935 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:40 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:21:40.935 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:40 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:21:41.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:40 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:41.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:40 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:21:41.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:40 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:21:41.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:40 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:21:41.003 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:40 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:41.003 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:40 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:21:41.003 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:40 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:21:41.003 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:40 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:21:41.823 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:41 smithi086 ceph-mon[22689]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T22:21:42.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:41 smithi067 ceph-mon[21875]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T22:21:42.003 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:41 smithi067 ceph-mon[26633]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T22:21:42.856 INFO:teuthology.orchestra.run.smithi086.stdout:Created osd(s) 4 on host 'smithi086' 2024-04-22T22:21:43.147 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:43 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:43.147 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:43 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:43.147 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:43 smithi086 ceph-mon[22689]: from='osd.4 [v2:172.21.15.86:6800/3080458560,v1:172.21.15.86:6801/3080458560]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T22:21:43.147 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:43 smithi086 ceph-mon[22689]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T22:21:43.147 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:43 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:43.147 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:43 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:21:43.147 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:43 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:21:43.147 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:43 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:21:43.253 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:43 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:43.253 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:43 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:43.253 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:43 smithi067 ceph-mon[21875]: from='osd.4 [v2:172.21.15.86:6800/3080458560,v1:172.21.15.86:6801/3080458560]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T22:21:43.253 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:43 smithi067 ceph-mon[21875]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T22:21:43.254 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:43 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:43.254 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:43 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:21:43.254 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:43 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:21:43.254 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:43 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:21:43.254 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:43 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:43.254 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:43 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:43.254 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:43 smithi067 ceph-mon[26633]: from='osd.4 [v2:172.21.15.86:6800/3080458560,v1:172.21.15.86:6801/3080458560]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T22:21:43.254 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:43 smithi067 ceph-mon[26633]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-22T22:21:43.254 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:43 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:43.254 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:43 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:21:43.254 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:43 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:21:43.254 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:43 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:21:43.431 DEBUG:teuthology.orchestra.run.smithi086:osd.4> sudo journalctl -f -n 0 -u ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@osd.4.service 2024-04-22T22:21:43.433 INFO:tasks.cephadm:Deploying osd.5 on smithi086 with /dev/vg_nvme/lv_3... 2024-04-22T22:21:43.433 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-22T22:21:44.252 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:44 smithi086 ceph-mon[22689]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T22:21:44.252 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:44 smithi086 ceph-mon[22689]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-22T22:21:44.252 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:44 smithi086 ceph-mon[22689]: osdmap e29: 5 total, 4 up, 5 in 2024-04-22T22:21:44.252 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:44 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T22:21:44.252 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:44 smithi086 ceph-mon[22689]: from='osd.4 [v2:172.21.15.86:6800/3080458560,v1:172.21.15.86:6801/3080458560]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-22T22:21:44.252 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:44 smithi086 ceph-mon[22689]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-22T22:21:44.253 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:44 smithi067 ceph-mon[21875]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T22:21:44.253 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:44 smithi067 ceph-mon[21875]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-22T22:21:44.253 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:44 smithi067 ceph-mon[21875]: osdmap e29: 5 total, 4 up, 5 in 2024-04-22T22:21:44.253 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:44 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T22:21:44.253 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:44 smithi067 ceph-mon[21875]: from='osd.4 [v2:172.21.15.86:6800/3080458560,v1:172.21.15.86:6801/3080458560]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-22T22:21:44.253 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:44 smithi067 ceph-mon[21875]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-22T22:21:44.253 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:44 smithi067 ceph-mon[26633]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T22:21:44.253 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:44 smithi067 ceph-mon[26633]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-22T22:21:44.254 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:44 smithi067 ceph-mon[26633]: osdmap e29: 5 total, 4 up, 5 in 2024-04-22T22:21:44.254 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:44 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T22:21:44.254 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:44 smithi067 ceph-mon[26633]: from='osd.4 [v2:172.21.15.86:6800/3080458560,v1:172.21.15.86:6801/3080458560]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-22T22:21:44.254 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:44 smithi067 ceph-mon[26633]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-22T22:21:44.752 INFO:journalctl@ceph.osd.4.smithi086.stdout:Apr 22 22:21:44 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-4[25948]: 2024-04-22T22:21:44.322+0000 7f26181e9700 -1 osd.4 0 waiting for initial osdmap 2024-04-22T22:21:44.752 INFO:journalctl@ceph.osd.4.smithi086.stdout:Apr 22 22:21:44 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-4[25948]: 2024-04-22T22:21:44.326+0000 7f2613b82700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T22:21:45.752 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:45 smithi086 ceph-mon[22689]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]': finished 2024-04-22T22:21:45.752 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:45 smithi086 ceph-mon[22689]: osdmap e30: 5 total, 4 up, 5 in 2024-04-22T22:21:45.753 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:45 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T22:21:45.753 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:45 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T22:21:45.753 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:45 smithi086 ceph-mon[22689]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T22:21:45.753 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:45 smithi086 ceph-mon[22689]: osd.4 [v2:172.21.15.86:6800/3080458560,v1:172.21.15.86:6801/3080458560] boot 2024-04-22T22:21:45.753 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:45 smithi086 ceph-mon[22689]: osdmap e31: 5 total, 5 up, 5 in 2024-04-22T22:21:45.753 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:45 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T22:21:45.753 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:45 smithi067 ceph-mon[21875]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]': finished 2024-04-22T22:21:45.753 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:45 smithi067 ceph-mon[21875]: osdmap e30: 5 total, 4 up, 5 in 2024-04-22T22:21:45.753 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:45 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T22:21:45.754 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:45 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T22:21:45.754 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:45 smithi067 ceph-mon[21875]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T22:21:45.754 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:45 smithi067 ceph-mon[21875]: osd.4 [v2:172.21.15.86:6800/3080458560,v1:172.21.15.86:6801/3080458560] boot 2024-04-22T22:21:45.754 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:45 smithi067 ceph-mon[21875]: osdmap e31: 5 total, 5 up, 5 in 2024-04-22T22:21:45.754 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:45 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T22:21:45.754 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:45 smithi067 ceph-mon[26633]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]': finished 2024-04-22T22:21:45.754 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:45 smithi067 ceph-mon[26633]: osdmap e30: 5 total, 4 up, 5 in 2024-04-22T22:21:45.754 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:45 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T22:21:45.754 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:45 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T22:21:45.754 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:45 smithi067 ceph-mon[26633]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-22T22:21:45.754 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:45 smithi067 ceph-mon[26633]: osd.4 [v2:172.21.15.86:6800/3080458560,v1:172.21.15.86:6801/3080458560] boot 2024-04-22T22:21:45.754 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:45 smithi067 ceph-mon[26633]: osdmap e31: 5 total, 5 up, 5 in 2024-04-22T22:21:45.754 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:45 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T22:21:45.770 INFO:teuthology.orchestra.run.smithi086.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-22T22:21:45.770 INFO:teuthology.orchestra.run.smithi086.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-22T22:21:45.770 INFO:teuthology.orchestra.run.smithi086.stderr: stderr: 10+0 records in 2024-04-22T22:21:45.770 INFO:teuthology.orchestra.run.smithi086.stderr:10+0 records out 2024-04-22T22:21:45.770 INFO:teuthology.orchestra.run.smithi086.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0124222 s, 844 MB/s 2024-04-22T22:21:45.770 INFO:teuthology.orchestra.run.smithi086.stderr:--> Zapping successful for: 2024-04-22T22:21:46.423 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi086:vg_nvme/lv_3 2024-04-22T22:21:46.651 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:46 smithi086 ceph-mon[22689]: purged_snaps scrub starts 2024-04-22T22:21:46.651 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:46 smithi086 ceph-mon[22689]: purged_snaps scrub ok 2024-04-22T22:21:46.651 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:46 smithi086 ceph-mon[22689]: osdmap e32: 5 total, 5 up, 5 in 2024-04-22T22:21:47.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:46 smithi067 ceph-mon[21875]: purged_snaps scrub starts 2024-04-22T22:21:47.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:46 smithi067 ceph-mon[21875]: purged_snaps scrub ok 2024-04-22T22:21:47.004 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:46 smithi067 ceph-mon[21875]: osdmap e32: 5 total, 5 up, 5 in 2024-04-22T22:21:47.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:46 smithi067 ceph-mon[26633]: purged_snaps scrub starts 2024-04-22T22:21:47.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:46 smithi067 ceph-mon[26633]: purged_snaps scrub ok 2024-04-22T22:21:47.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:46 smithi067 ceph-mon[26633]: osdmap e32: 5 total, 5 up, 5 in 2024-04-22T22:21:47.834 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:47 smithi086 ceph-mon[22689]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-22T22:21:47.834 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:47 smithi086 ceph-mon[22689]: osdmap e33: 5 total, 5 up, 5 in 2024-04-22T22:21:48.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:47 smithi067 ceph-mon[21875]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-22T22:21:48.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:47 smithi067 ceph-mon[21875]: osdmap e33: 5 total, 5 up, 5 in 2024-04-22T22:21:48.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:47 smithi067 ceph-mon[26633]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-22T22:21:48.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:47 smithi067 ceph-mon[26633]: osdmap e33: 5 total, 5 up, 5 in 2024-04-22T22:21:48.752 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:48 smithi086 ceph-mon[22689]: osdmap e34: 5 total, 5 up, 5 in 2024-04-22T22:21:48.752 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:48 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:48.752 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:48 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:21:48.752 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:48 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:48.752 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:48 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:48.752 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:48 smithi086 ceph-mon[22689]: osdmap e35: 5 total, 5 up, 5 in 2024-04-22T22:21:49.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:48 smithi067 ceph-mon[21875]: osdmap e34: 5 total, 5 up, 5 in 2024-04-22T22:21:49.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:48 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:49.004 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:48 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:21:49.004 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:48 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:49.004 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:48 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:49.004 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:48 smithi067 ceph-mon[21875]: osdmap e35: 5 total, 5 up, 5 in 2024-04-22T22:21:49.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:48 smithi067 ceph-mon[26633]: osdmap e34: 5 total, 5 up, 5 in 2024-04-22T22:21:49.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:48 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:49.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:48 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:21:49.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:48 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:49.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:48 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:21:49.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:48 smithi067 ceph-mon[26633]: osdmap e35: 5 total, 5 up, 5 in 2024-04-22T22:21:49.843 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:49 smithi086 ceph-mon[22689]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-22T22:21:49.843 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:49 smithi086 ceph-mon[22689]: Detected new or changed devices on smithi086 2024-04-22T22:21:49.843 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:49 smithi086 ceph-mon[22689]: Adjusting osd_memory_target on smithi086 to 17088M 2024-04-22T22:21:49.843 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:49 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T22:21:49.844 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:49 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T22:21:49.844 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:49 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:21:50.004 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:49 smithi067 ceph-mon[21875]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-22T22:21:50.004 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:49 smithi067 ceph-mon[21875]: Detected new or changed devices on smithi086 2024-04-22T22:21:50.004 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:49 smithi067 ceph-mon[21875]: Adjusting osd_memory_target on smithi086 to 17088M 2024-04-22T22:21:50.004 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:49 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T22:21:50.004 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:49 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T22:21:50.004 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:49 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:21:50.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:49 smithi067 ceph-mon[26633]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-22T22:21:50.005 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:49 smithi067 ceph-mon[26633]: Detected new or changed devices on smithi086 2024-04-22T22:21:50.005 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:49 smithi067 ceph-mon[26633]: Adjusting osd_memory_target on smithi086 to 17088M 2024-04-22T22:21:50.005 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:49 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T22:21:50.005 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:49 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T22:21:50.005 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:49 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:21:50.752 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:50 smithi086 ceph-mon[22689]: from='client.24239 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi086:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:21:50.905 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:50 smithi067 ceph-mon[21875]: from='client.24239 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi086:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:21:50.905 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:50 smithi067 ceph-mon[26633]: from='client.24239 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi086:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:21:51.752 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:51 smithi086 ceph-mon[22689]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 106 KiB/s, 0 objects/s recovering 2024-04-22T22:21:51.752 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:51 smithi086 ceph-mon[22689]: from='client.? 172.21.15.86:0/2326702053' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a7412a3c-3fb5-4632-a91a-7daf2c5b221e"}]: dispatch 2024-04-22T22:21:51.752 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:51 smithi086 ceph-mon[22689]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a7412a3c-3fb5-4632-a91a-7daf2c5b221e"}]: dispatch 2024-04-22T22:21:51.752 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:51 smithi086 ceph-mon[22689]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a7412a3c-3fb5-4632-a91a-7daf2c5b221e"}]': finished 2024-04-22T22:21:51.752 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:51 smithi086 ceph-mon[22689]: osdmap e36: 6 total, 5 up, 6 in 2024-04-22T22:21:51.752 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:51 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T22:21:52.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:51 smithi067 ceph-mon[21875]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 106 KiB/s, 0 objects/s recovering 2024-04-22T22:21:52.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:51 smithi067 ceph-mon[21875]: from='client.? 172.21.15.86:0/2326702053' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a7412a3c-3fb5-4632-a91a-7daf2c5b221e"}]: dispatch 2024-04-22T22:21:52.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:51 smithi067 ceph-mon[21875]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a7412a3c-3fb5-4632-a91a-7daf2c5b221e"}]: dispatch 2024-04-22T22:21:52.004 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:51 smithi067 ceph-mon[21875]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a7412a3c-3fb5-4632-a91a-7daf2c5b221e"}]': finished 2024-04-22T22:21:52.004 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:51 smithi067 ceph-mon[21875]: osdmap e36: 6 total, 5 up, 6 in 2024-04-22T22:21:52.004 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:51 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T22:21:52.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:51 smithi067 ceph-mon[26633]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 106 KiB/s, 0 objects/s recovering 2024-04-22T22:21:52.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:51 smithi067 ceph-mon[26633]: from='client.? 172.21.15.86:0/2326702053' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a7412a3c-3fb5-4632-a91a-7daf2c5b221e"}]: dispatch 2024-04-22T22:21:52.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:51 smithi067 ceph-mon[26633]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a7412a3c-3fb5-4632-a91a-7daf2c5b221e"}]: dispatch 2024-04-22T22:21:52.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:51 smithi067 ceph-mon[26633]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a7412a3c-3fb5-4632-a91a-7daf2c5b221e"}]': finished 2024-04-22T22:21:52.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:51 smithi067 ceph-mon[26633]: osdmap e36: 6 total, 5 up, 6 in 2024-04-22T22:21:52.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:51 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T22:21:53.002 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:52 smithi086 ceph-mon[22689]: from='client.? 172.21.15.86:0/116224678' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T22:21:53.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:52 smithi067 ceph-mon[21875]: from='client.? 172.21.15.86:0/116224678' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T22:21:53.003 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:52 smithi067 ceph-mon[26633]: from='client.? 172.21.15.86:0/116224678' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T22:21:54.002 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:53 smithi086 ceph-mon[22689]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 88 KiB/s, 0 objects/s recovering 2024-04-22T22:21:54.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:53 smithi067 ceph-mon[21875]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 88 KiB/s, 0 objects/s recovering 2024-04-22T22:21:54.003 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:53 smithi067 ceph-mon[26633]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 88 KiB/s, 0 objects/s recovering 2024-04-22T22:21:55.936 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:55 smithi086 ceph-mon[22689]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-04-22T22:21:56.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:55 smithi067 ceph-mon[21875]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-04-22T22:21:56.003 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:55 smithi067 ceph-mon[26633]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-04-22T22:21:57.804 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:57 smithi086 ceph-mon[22689]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-22T22:21:57.805 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:57 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-22T22:21:57.805 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:57 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:21:58.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:57 smithi067 ceph-mon[21875]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-22T22:21:58.004 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:57 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-22T22:21:58.004 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:57 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:21:58.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:57 smithi067 ceph-mon[26633]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-22T22:21:58.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:57 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-22T22:21:58.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:57 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:21:58.866 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:58 smithi067 ceph-mon[21875]: Deploying daemon osd.5 on smithi086 2024-04-22T22:21:58.866 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:58 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:21:58.867 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:58 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:21:58.867 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:58 smithi067 ceph-mon[26633]: Deploying daemon osd.5 on smithi086 2024-04-22T22:21:58.867 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:58 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:21:58.867 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:58 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:21:58.934 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:58 smithi086 ceph-mon[22689]: Deploying daemon osd.5 on smithi086 2024-04-22T22:21:58.935 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:58 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:21:58.935 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:58 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:21:59.986 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:21:59 smithi086 ceph-mon[22689]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-22T22:22:00.004 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:21:59 smithi067 ceph-mon[21875]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-22T22:22:00.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:21:59 smithi067 ceph-mon[26633]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-22T22:22:01.807 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:01 smithi086 ceph-mon[22689]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T22:22:02.004 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:01 smithi067 ceph-mon[21875]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T22:22:02.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:01 smithi067 ceph-mon[26633]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T22:22:02.752 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:02 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:22:02.752 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:02 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:02.753 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:02 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:22:02.753 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:02 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:22:03.004 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:02 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:22:03.004 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:02 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:03.004 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:02 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:22:03.004 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:02 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:22:03.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:02 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:22:03.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:02 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:03.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:02 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:22:03.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:02 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:22:03.753 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:03 smithi086 ceph-mon[22689]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T22:22:04.004 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:03 smithi067 ceph-mon[21875]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T22:22:04.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:03 smithi067 ceph-mon[26633]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T22:22:05.349 INFO:teuthology.orchestra.run.smithi086.stdout:Created osd(s) 5 on host 'smithi086' 2024-04-22T22:22:05.649 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:05 smithi086 ceph-mon[22689]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T22:22:05.650 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:05 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:05.650 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:05 smithi086 ceph-mon[22689]: from='osd.5 [v2:172.21.15.86:6808/451381933,v1:172.21.15.86:6809/451381933]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T22:22:05.650 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:05 smithi086 ceph-mon[22689]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T22:22:05.964 DEBUG:teuthology.orchestra.run.smithi086:osd.5> sudo journalctl -f -n 0 -u ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@osd.5.service 2024-04-22T22:22:05.967 INFO:tasks.cephadm:Deploying osd.6 on smithi086 with /dev/vg_nvme/lv_2... 2024-04-22T22:22:05.967 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-22T22:22:06.003 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:05 smithi067 ceph-mon[21875]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T22:22:06.004 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:05 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:06.004 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:05 smithi067 ceph-mon[21875]: from='osd.5 [v2:172.21.15.86:6808/451381933,v1:172.21.15.86:6809/451381933]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T22:22:06.004 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:05 smithi067 ceph-mon[21875]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T22:22:06.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:05 smithi067 ceph-mon[26633]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T22:22:06.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:05 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:06.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:05 smithi067 ceph-mon[26633]: from='osd.5 [v2:172.21.15.86:6808/451381933,v1:172.21.15.86:6809/451381933]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T22:22:06.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:05 smithi067 ceph-mon[26633]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-22T22:22:07.252 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:07 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:07.253 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:07 smithi086 ceph-mon[22689]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-22T22:22:07.253 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:07 smithi086 ceph-mon[22689]: osdmap e37: 6 total, 5 up, 6 in 2024-04-22T22:22:07.253 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:07 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T22:22:07.253 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:07 smithi086 ceph-mon[22689]: from='osd.5 [v2:172.21.15.86:6808/451381933,v1:172.21.15.86:6809/451381933]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-22T22:22:07.253 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:07 smithi086 ceph-mon[22689]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-22T22:22:07.504 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:07 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:07.504 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:07 smithi067 ceph-mon[21875]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-22T22:22:07.504 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:07 smithi067 ceph-mon[21875]: osdmap e37: 6 total, 5 up, 6 in 2024-04-22T22:22:07.504 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:07 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T22:22:07.504 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:07 smithi067 ceph-mon[21875]: from='osd.5 [v2:172.21.15.86:6808/451381933,v1:172.21.15.86:6809/451381933]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-22T22:22:07.504 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:07 smithi067 ceph-mon[21875]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-22T22:22:07.504 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:07 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:07.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:07 smithi067 ceph-mon[26633]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-22T22:22:07.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:07 smithi067 ceph-mon[26633]: osdmap e37: 6 total, 5 up, 6 in 2024-04-22T22:22:07.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:07 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T22:22:07.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:07 smithi067 ceph-mon[26633]: from='osd.5 [v2:172.21.15.86:6808/451381933,v1:172.21.15.86:6809/451381933]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-22T22:22:07.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:07 smithi067 ceph-mon[26633]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-22T22:22:07.663 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:22:07 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-5[28891]: 2024-04-22T22:22:07.376+0000 7f63a7c9f700 -1 osd.5 0 waiting for initial osdmap 2024-04-22T22:22:07.663 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:22:07 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-5[28891]: 2024-04-22T22:22:07.381+0000 7f63a0e33700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T22:22:08.209 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:08 smithi086 ceph-mon[22689]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T22:22:08.210 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:08 smithi086 ceph-mon[22689]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]': finished 2024-04-22T22:22:08.210 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:08 smithi086 ceph-mon[22689]: osdmap e38: 6 total, 5 up, 6 in 2024-04-22T22:22:08.210 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:08 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T22:22:08.210 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:08 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T22:22:08.504 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:08 smithi067 ceph-mon[21875]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T22:22:08.504 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:08 smithi067 ceph-mon[21875]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]': finished 2024-04-22T22:22:08.504 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:08 smithi067 ceph-mon[21875]: osdmap e38: 6 total, 5 up, 6 in 2024-04-22T22:22:08.504 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:08 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T22:22:08.504 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:08 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T22:22:08.504 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:08 smithi067 ceph-mon[26633]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-22T22:22:08.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:08 smithi067 ceph-mon[26633]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]': finished 2024-04-22T22:22:08.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:08 smithi067 ceph-mon[26633]: osdmap e38: 6 total, 5 up, 6 in 2024-04-22T22:22:08.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:08 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T22:22:08.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:08 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T22:22:08.550 INFO:teuthology.orchestra.run.smithi086.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-22T22:22:08.551 INFO:teuthology.orchestra.run.smithi086.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-22T22:22:08.551 INFO:teuthology.orchestra.run.smithi086.stderr: stderr: 10+0 records in 2024-04-22T22:22:08.551 INFO:teuthology.orchestra.run.smithi086.stderr:10+0 records out 2024-04-22T22:22:08.551 INFO:teuthology.orchestra.run.smithi086.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0123513 s, 849 MB/s 2024-04-22T22:22:08.551 INFO:teuthology.orchestra.run.smithi086.stderr:--> Zapping successful for: 2024-04-22T22:22:09.568 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:09 smithi086 ceph-mon[22689]: purged_snaps scrub starts 2024-04-22T22:22:09.568 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:09 smithi086 ceph-mon[22689]: purged_snaps scrub ok 2024-04-22T22:22:09.569 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:09 smithi086 ceph-mon[22689]: osd.5 [v2:172.21.15.86:6808/451381933,v1:172.21.15.86:6809/451381933] boot 2024-04-22T22:22:09.569 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:09 smithi086 ceph-mon[22689]: osdmap e39: 6 total, 6 up, 6 in 2024-04-22T22:22:09.569 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:09 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T22:22:09.569 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:09 smithi086 ceph-mon[22689]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-22T22:22:09.744 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi086:vg_nvme/lv_2 2024-04-22T22:22:09.754 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:09 smithi067 ceph-mon[21875]: purged_snaps scrub starts 2024-04-22T22:22:09.754 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:09 smithi067 ceph-mon[21875]: purged_snaps scrub ok 2024-04-22T22:22:09.754 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:09 smithi067 ceph-mon[21875]: osd.5 [v2:172.21.15.86:6808/451381933,v1:172.21.15.86:6809/451381933] boot 2024-04-22T22:22:09.754 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:09 smithi067 ceph-mon[21875]: osdmap e39: 6 total, 6 up, 6 in 2024-04-22T22:22:09.754 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:09 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T22:22:09.754 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:09 smithi067 ceph-mon[21875]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-22T22:22:09.754 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:09 smithi067 ceph-mon[26633]: purged_snaps scrub starts 2024-04-22T22:22:09.755 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:09 smithi067 ceph-mon[26633]: purged_snaps scrub ok 2024-04-22T22:22:09.755 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:09 smithi067 ceph-mon[26633]: osd.5 [v2:172.21.15.86:6808/451381933,v1:172.21.15.86:6809/451381933] boot 2024-04-22T22:22:09.755 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:09 smithi067 ceph-mon[26633]: osdmap e39: 6 total, 6 up, 6 in 2024-04-22T22:22:09.755 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:09 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T22:22:09.755 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:09 smithi067 ceph-mon[26633]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-22T22:22:10.422 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:10 smithi086 ceph-mon[22689]: osdmap e40: 6 total, 6 up, 6 in 2024-04-22T22:22:10.754 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:10 smithi067 ceph-mon[21875]: osdmap e40: 6 total, 6 up, 6 in 2024-04-22T22:22:10.754 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:10 smithi067 ceph-mon[26633]: osdmap e40: 6 total, 6 up, 6 in 2024-04-22T22:22:11.657 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:11 smithi086 ceph-mon[22689]: osdmap e41: 6 total, 6 up, 6 in 2024-04-22T22:22:11.657 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:11 smithi086 ceph-mon[22689]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-22T22:22:11.657 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:11 smithi086 ceph-mon[22689]: Detected new or changed devices on smithi086 2024-04-22T22:22:11.657 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:11 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:11.657 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:11 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:22:11.657 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:11 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:22:11.657 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:11 smithi086 ceph-mon[22689]: Adjusting osd_memory_target on smithi086 to 8544M 2024-04-22T22:22:11.657 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:11 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:11.657 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:11 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:11.658 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:11 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:22:11.658 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:11 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:22:11.658 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:11 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:22:11.658 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:11 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:11.754 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:11 smithi067 ceph-mon[21875]: osdmap e41: 6 total, 6 up, 6 in 2024-04-22T22:22:11.754 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:11 smithi067 ceph-mon[21875]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-22T22:22:11.754 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:11 smithi067 ceph-mon[21875]: Detected new or changed devices on smithi086 2024-04-22T22:22:11.754 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:11 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:11.754 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:11 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:22:11.754 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:11 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:22:11.754 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:11 smithi067 ceph-mon[21875]: Adjusting osd_memory_target on smithi086 to 8544M 2024-04-22T22:22:11.754 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:11 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:11.755 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:11 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:11.755 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:11 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:22:11.755 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:11 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:22:11.755 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:11 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:22:11.755 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:11 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:11.755 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:11 smithi067 ceph-mon[26633]: osdmap e41: 6 total, 6 up, 6 in 2024-04-22T22:22:11.755 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:11 smithi067 ceph-mon[26633]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-22T22:22:11.755 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:11 smithi067 ceph-mon[26633]: Detected new or changed devices on smithi086 2024-04-22T22:22:11.755 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:11 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:11.755 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:11 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:22:11.755 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:11 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:22:11.755 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:11 smithi067 ceph-mon[26633]: Adjusting osd_memory_target on smithi086 to 8544M 2024-04-22T22:22:11.755 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:11 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:11.755 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:11 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:11.755 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:11 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:22:11.756 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:11 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:22:11.756 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:11 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:22:11.756 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:11 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:12.502 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:12 smithi086 ceph-mon[22689]: osdmap e42: 6 total, 6 up, 6 in 2024-04-22T22:22:12.503 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:12 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T22:22:12.503 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:12 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T22:22:12.503 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:12 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:22:12.754 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:12 smithi067 ceph-mon[21875]: osdmap e42: 6 total, 6 up, 6 in 2024-04-22T22:22:12.754 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:12 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T22:22:12.754 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:12 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T22:22:12.754 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:12 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:22:12.754 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:12 smithi067 ceph-mon[26633]: osdmap e42: 6 total, 6 up, 6 in 2024-04-22T22:22:12.755 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:12 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T22:22:12.755 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:12 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T22:22:12.755 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:12 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:22:13.719 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:13 smithi086 ceph-mon[22689]: from='client.24266 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi086:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:22:13.719 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:13 smithi086 ceph-mon[22689]: pgmap v129: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-22T22:22:13.719 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:13 smithi086 ceph-mon[22689]: osdmap e43: 6 total, 6 up, 6 in 2024-04-22T22:22:13.754 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:13 smithi067 ceph-mon[21875]: from='client.24266 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi086:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:22:13.754 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:13 smithi067 ceph-mon[21875]: pgmap v129: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-22T22:22:13.754 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:13 smithi067 ceph-mon[21875]: osdmap e43: 6 total, 6 up, 6 in 2024-04-22T22:22:13.754 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:13 smithi067 ceph-mon[26633]: from='client.24266 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi086:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:22:13.754 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:13 smithi067 ceph-mon[26633]: pgmap v129: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-22T22:22:13.754 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:13 smithi067 ceph-mon[26633]: osdmap e43: 6 total, 6 up, 6 in 2024-04-22T22:22:14.753 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:14 smithi086 ceph-mon[22689]: from='client.? 172.21.15.86:0/3101231048' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d54fd043-fcec-437d-bd01-a07f71f97430"}]: dispatch 2024-04-22T22:22:14.753 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:14 smithi086 ceph-mon[22689]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d54fd043-fcec-437d-bd01-a07f71f97430"}]: dispatch 2024-04-22T22:22:14.753 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:14 smithi086 ceph-mon[22689]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d54fd043-fcec-437d-bd01-a07f71f97430"}]': finished 2024-04-22T22:22:14.753 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:14 smithi086 ceph-mon[22689]: osdmap e44: 7 total, 6 up, 7 in 2024-04-22T22:22:14.753 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:14 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T22:22:14.754 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:14 smithi067 ceph-mon[21875]: from='client.? 172.21.15.86:0/3101231048' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d54fd043-fcec-437d-bd01-a07f71f97430"}]: dispatch 2024-04-22T22:22:14.754 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:14 smithi067 ceph-mon[21875]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d54fd043-fcec-437d-bd01-a07f71f97430"}]: dispatch 2024-04-22T22:22:14.754 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:14 smithi067 ceph-mon[21875]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d54fd043-fcec-437d-bd01-a07f71f97430"}]': finished 2024-04-22T22:22:14.755 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:14 smithi067 ceph-mon[21875]: osdmap e44: 7 total, 6 up, 7 in 2024-04-22T22:22:14.755 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:14 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T22:22:14.755 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:14 smithi067 ceph-mon[26633]: from='client.? 172.21.15.86:0/3101231048' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d54fd043-fcec-437d-bd01-a07f71f97430"}]: dispatch 2024-04-22T22:22:14.756 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:14 smithi067 ceph-mon[26633]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d54fd043-fcec-437d-bd01-a07f71f97430"}]: dispatch 2024-04-22T22:22:14.756 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:14 smithi067 ceph-mon[26633]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d54fd043-fcec-437d-bd01-a07f71f97430"}]': finished 2024-04-22T22:22:14.756 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:14 smithi067 ceph-mon[26633]: osdmap e44: 7 total, 6 up, 7 in 2024-04-22T22:22:14.756 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:14 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T22:22:16.003 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:15 smithi086 ceph-mon[22689]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T22:22:16.003 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:15 smithi086 ceph-mon[22689]: from='client.? 172.21.15.86:0/3960372720' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T22:22:16.004 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:15 smithi067 ceph-mon[21875]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T22:22:16.004 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:15 smithi067 ceph-mon[21875]: from='client.? 172.21.15.86:0/3960372720' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T22:22:16.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:15 smithi067 ceph-mon[26633]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T22:22:16.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:15 smithi067 ceph-mon[26633]: from='client.? 172.21.15.86:0/3960372720' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T22:22:17.818 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:17 smithi086 ceph-mon[22689]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T22:22:18.004 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:17 smithi067 ceph-mon[21875]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T22:22:18.005 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:17 smithi067 ceph-mon[26633]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T22:22:19.753 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:19 smithi086 ceph-mon[22689]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T22:22:20.004 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:19 smithi067 ceph-mon[21875]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T22:22:20.005 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:19 smithi067 ceph-mon[26633]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T22:22:20.707 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:20 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-22T22:22:20.707 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:20 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:22:20.905 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:20 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-22T22:22:20.906 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:20 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:22:20.906 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:20 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-22T22:22:20.906 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:20 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:22:21.753 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:21 smithi086 ceph-mon[22689]: Deploying daemon osd.6 on smithi086 2024-04-22T22:22:21.753 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:21 smithi086 ceph-mon[22689]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T22:22:22.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:21 smithi067 ceph-mon[26633]: Deploying daemon osd.6 on smithi086 2024-04-22T22:22:22.004 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:21 smithi067 ceph-mon[26633]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T22:22:22.005 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:21 smithi067 ceph-mon[21875]: Deploying daemon osd.6 on smithi086 2024-04-22T22:22:22.005 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:21 smithi067 ceph-mon[21875]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T22:22:23.878 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:23 smithi086 ceph-mon[22689]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T22:22:24.004 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:23 smithi067 ceph-mon[21875]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T22:22:24.005 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:23 smithi067 ceph-mon[26633]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T22:22:26.241 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:26 smithi086 ceph-mon[22689]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T22:22:26.242 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:26 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:26.242 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:26 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:22:26.242 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:26 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:22:26.242 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:26 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:22:26.254 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:26 smithi067 ceph-mon[26633]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T22:22:26.254 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:26 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:26.255 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:26 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:22:26.255 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:26 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:22:26.255 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:26 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:22:26.255 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:26 smithi067 ceph-mon[21875]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T22:22:26.255 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:26 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:26.255 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:26 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:22:26.255 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:26 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:22:26.255 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:26 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:22:28.192 INFO:teuthology.orchestra.run.smithi086.stdout:Created osd(s) 6 on host 'smithi086' 2024-04-22T22:22:28.253 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:28 smithi086 ceph-mon[22689]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T22:22:28.253 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:28 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:28.253 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:28 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:28.254 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:28 smithi067 ceph-mon[21875]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T22:22:28.255 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:28 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:28.255 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:28 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:28.255 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:28 smithi067 ceph-mon[26633]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T22:22:28.255 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:28 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:28.255 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:28 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:28.867 DEBUG:teuthology.orchestra.run.smithi086:osd.6> sudo journalctl -f -n 0 -u ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@osd.6.service 2024-04-22T22:22:28.869 INFO:tasks.cephadm:Deploying osd.7 on smithi086 with /dev/vg_nvme/lv_1... 2024-04-22T22:22:28.869 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-22T22:22:29.137 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:29 smithi086 ceph-mon[22689]: from='osd.6 [v2:172.21.15.86:6816/2900445442,v1:172.21.15.86:6817/2900445442]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T22:22:29.137 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:29 smithi086 ceph-mon[22689]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T22:22:29.137 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:29 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:22:29.137 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:29 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:29.137 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:29 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:22:29.137 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:29 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:22:29.255 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:29 smithi067 ceph-mon[21875]: from='osd.6 [v2:172.21.15.86:6816/2900445442,v1:172.21.15.86:6817/2900445442]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T22:22:29.255 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:29 smithi067 ceph-mon[21875]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T22:22:29.255 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:29 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:22:29.255 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:29 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:29.255 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:29 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:22:29.255 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:29 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:22:29.255 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:29 smithi067 ceph-mon[26633]: from='osd.6 [v2:172.21.15.86:6816/2900445442,v1:172.21.15.86:6817/2900445442]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T22:22:29.255 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:29 smithi067 ceph-mon[26633]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-22T22:22:29.255 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:29 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:22:29.255 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:29 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:29.256 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:29 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:22:29.256 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:29 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:22:30.356 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:30 smithi086 ceph-mon[22689]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T22:22:30.357 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:30 smithi086 ceph-mon[22689]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-22T22:22:30.357 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:30 smithi086 ceph-mon[22689]: osdmap e45: 7 total, 6 up, 7 in 2024-04-22T22:22:30.357 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:30 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T22:22:30.357 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:30 smithi086 ceph-mon[22689]: from='osd.6 [v2:172.21.15.86:6816/2900445442,v1:172.21.15.86:6817/2900445442]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-22T22:22:30.357 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:30 smithi086 ceph-mon[22689]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-22T22:22:30.357 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:22:30 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:22:30.027+0000 7fcfb52d1700 -1 osd.6 0 waiting for initial osdmap 2024-04-22T22:22:30.357 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:22:30 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:22:30.034+0000 7fcfb0469700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T22:22:30.505 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:30 smithi067 ceph-mon[21875]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T22:22:30.505 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:30 smithi067 ceph-mon[21875]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-22T22:22:30.505 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:30 smithi067 ceph-mon[21875]: osdmap e45: 7 total, 6 up, 7 in 2024-04-22T22:22:30.505 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:30 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T22:22:30.505 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:30 smithi067 ceph-mon[21875]: from='osd.6 [v2:172.21.15.86:6816/2900445442,v1:172.21.15.86:6817/2900445442]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-22T22:22:30.505 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:30 smithi067 ceph-mon[21875]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-22T22:22:30.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:30 smithi067 ceph-mon[26633]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T22:22:30.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:30 smithi067 ceph-mon[26633]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-22T22:22:30.506 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:30 smithi067 ceph-mon[26633]: osdmap e45: 7 total, 6 up, 7 in 2024-04-22T22:22:30.506 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:30 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T22:22:30.506 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:30 smithi067 ceph-mon[26633]: from='osd.6 [v2:172.21.15.86:6816/2900445442,v1:172.21.15.86:6817/2900445442]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-22T22:22:30.506 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:30 smithi067 ceph-mon[26633]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-22T22:22:31.253 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:31 smithi086 ceph-mon[22689]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]': finished 2024-04-22T22:22:31.254 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:31 smithi086 ceph-mon[22689]: osdmap e46: 7 total, 6 up, 7 in 2024-04-22T22:22:31.254 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:31 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T22:22:31.254 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:31 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T22:22:31.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:31 smithi067 ceph-mon[26633]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]': finished 2024-04-22T22:22:31.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:31 smithi067 ceph-mon[26633]: osdmap e46: 7 total, 6 up, 7 in 2024-04-22T22:22:31.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:31 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T22:22:31.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:31 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T22:22:31.505 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:31 smithi067 ceph-mon[21875]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]': finished 2024-04-22T22:22:31.505 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:31 smithi067 ceph-mon[21875]: osdmap e46: 7 total, 6 up, 7 in 2024-04-22T22:22:31.505 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:31 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T22:22:31.505 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:31 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T22:22:32.343 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:32 smithi086 ceph-mon[22689]: purged_snaps scrub starts 2024-04-22T22:22:32.343 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:32 smithi086 ceph-mon[22689]: purged_snaps scrub ok 2024-04-22T22:22:32.343 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:32 smithi086 ceph-mon[22689]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T22:22:32.343 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:32 smithi086 ceph-mon[22689]: osd.6 [v2:172.21.15.86:6816/2900445442,v1:172.21.15.86:6817/2900445442] boot 2024-04-22T22:22:32.343 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:32 smithi086 ceph-mon[22689]: osdmap e47: 7 total, 7 up, 7 in 2024-04-22T22:22:32.343 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:32 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T22:22:32.343 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:32 smithi086 ceph-mon[22689]: osdmap e48: 7 total, 7 up, 7 in 2024-04-22T22:22:32.504 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:32 smithi067 ceph-mon[26633]: purged_snaps scrub starts 2024-04-22T22:22:32.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:32 smithi067 ceph-mon[26633]: purged_snaps scrub ok 2024-04-22T22:22:32.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:32 smithi067 ceph-mon[26633]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T22:22:32.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:32 smithi067 ceph-mon[26633]: osd.6 [v2:172.21.15.86:6816/2900445442,v1:172.21.15.86:6817/2900445442] boot 2024-04-22T22:22:32.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:32 smithi067 ceph-mon[26633]: osdmap e47: 7 total, 7 up, 7 in 2024-04-22T22:22:32.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:32 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T22:22:32.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:32 smithi067 ceph-mon[26633]: osdmap e48: 7 total, 7 up, 7 in 2024-04-22T22:22:32.505 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:32 smithi067 ceph-mon[21875]: purged_snaps scrub starts 2024-04-22T22:22:32.505 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:32 smithi067 ceph-mon[21875]: purged_snaps scrub ok 2024-04-22T22:22:32.505 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:32 smithi067 ceph-mon[21875]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-22T22:22:32.505 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:32 smithi067 ceph-mon[21875]: osd.6 [v2:172.21.15.86:6816/2900445442,v1:172.21.15.86:6817/2900445442] boot 2024-04-22T22:22:32.505 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:32 smithi067 ceph-mon[21875]: osdmap e47: 7 total, 7 up, 7 in 2024-04-22T22:22:32.506 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:32 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T22:22:32.506 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:32 smithi067 ceph-mon[21875]: osdmap e48: 7 total, 7 up, 7 in 2024-04-22T22:22:32.707 INFO:teuthology.orchestra.run.smithi086.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-22T22:22:32.707 INFO:teuthology.orchestra.run.smithi086.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-22T22:22:32.707 INFO:teuthology.orchestra.run.smithi086.stderr: stderr: 10+0 records in 2024-04-22T22:22:32.707 INFO:teuthology.orchestra.run.smithi086.stderr:10+0 records out 2024-04-22T22:22:32.707 INFO:teuthology.orchestra.run.smithi086.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0136975 s, 766 MB/s 2024-04-22T22:22:32.707 INFO:teuthology.orchestra.run.smithi086.stderr:--> Zapping successful for: 2024-04-22T22:22:33.340 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:33 smithi086 ceph-mon[22689]: Detected new or changed devices on smithi086 2024-04-22T22:22:33.340 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:33 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:33.340 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:33 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:22:33.341 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:33 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:22:33.341 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:33 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:22:33.341 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:33 smithi086 ceph-mon[22689]: Adjusting osd_memory_target on smithi086 to 5696M 2024-04-22T22:22:33.341 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:33 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:33.341 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:33 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:33.341 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:33 smithi086 ceph-mon[22689]: pgmap v145: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-22T22:22:33.341 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:33 smithi086 ceph-mon[22689]: osdmap e49: 7 total, 7 up, 7 in 2024-04-22T22:22:33.364 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi086:vg_nvme/lv_1 2024-04-22T22:22:33.755 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:33 smithi067 ceph-mon[21875]: Detected new or changed devices on smithi086 2024-04-22T22:22:33.755 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:33 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:33.755 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:33 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:22:33.755 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:33 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:22:33.755 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:33 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:22:33.755 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:33 smithi067 ceph-mon[21875]: Adjusting osd_memory_target on smithi086 to 5696M 2024-04-22T22:22:33.755 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:33 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:33.755 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:33 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:33.755 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:33 smithi067 ceph-mon[21875]: pgmap v145: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-22T22:22:33.755 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:33 smithi067 ceph-mon[21875]: osdmap e49: 7 total, 7 up, 7 in 2024-04-22T22:22:33.755 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:33 smithi067 ceph-mon[26633]: Detected new or changed devices on smithi086 2024-04-22T22:22:33.756 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:33 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:33.756 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:33 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:22:33.756 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:33 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:22:33.756 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:33 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:22:33.756 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:33 smithi067 ceph-mon[26633]: Adjusting osd_memory_target on smithi086 to 5696M 2024-04-22T22:22:33.756 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:33 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:33.756 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:33 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:33.756 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:33 smithi067 ceph-mon[26633]: pgmap v145: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-22T22:22:33.756 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:33 smithi067 ceph-mon[26633]: osdmap e49: 7 total, 7 up, 7 in 2024-04-22T22:22:35.160 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:35 smithi086 ceph-mon[22689]: osdmap e50: 7 total, 7 up, 7 in 2024-04-22T22:22:35.505 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:35 smithi067 ceph-mon[21875]: osdmap e50: 7 total, 7 up, 7 in 2024-04-22T22:22:35.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:35 smithi067 ceph-mon[26633]: osdmap e50: 7 total, 7 up, 7 in 2024-04-22T22:22:36.136 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:36 smithi086 ceph-mon[22689]: pgmap v148: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-22T22:22:36.136 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:36 smithi086 ceph-mon[22689]: osdmap e51: 7 total, 7 up, 7 in 2024-04-22T22:22:36.136 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:36 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T22:22:36.136 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:36 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T22:22:36.137 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:36 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:22:36.505 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:36 smithi067 ceph-mon[21875]: pgmap v148: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-22T22:22:36.505 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:36 smithi067 ceph-mon[21875]: osdmap e51: 7 total, 7 up, 7 in 2024-04-22T22:22:36.505 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:36 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T22:22:36.505 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:36 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T22:22:36.505 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:36 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:22:36.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:36 smithi067 ceph-mon[26633]: pgmap v148: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-22T22:22:36.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:36 smithi067 ceph-mon[26633]: osdmap e51: 7 total, 7 up, 7 in 2024-04-22T22:22:36.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:36 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-22T22:22:36.506 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:36 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-22T22:22:36.506 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:36 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:22:37.504 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:37 smithi086 ceph-mon[22689]: from='client.24293 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi086:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:22:37.504 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:37 smithi067 ceph-mon[21875]: from='client.24293 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi086:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:22:37.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:37 smithi067 ceph-mon[26633]: from='client.24293 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi086:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:22:38.503 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:38 smithi086 ceph-mon[22689]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 117 KiB/s, 0 objects/s recovering 2024-04-22T22:22:38.504 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:38 smithi086 ceph-mon[22689]: from='client.? 172.21.15.86:0/2201254328' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dc347749-843b-4fa9-9ee5-d4fd966e3c41"}]: dispatch 2024-04-22T22:22:38.504 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:38 smithi086 ceph-mon[22689]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dc347749-843b-4fa9-9ee5-d4fd966e3c41"}]: dispatch 2024-04-22T22:22:38.504 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:38 smithi086 ceph-mon[22689]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "dc347749-843b-4fa9-9ee5-d4fd966e3c41"}]': finished 2024-04-22T22:22:38.504 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:38 smithi086 ceph-mon[22689]: osdmap e52: 8 total, 7 up, 8 in 2024-04-22T22:22:38.504 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:38 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T22:22:38.504 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:38 smithi086 ceph-mon[22689]: from='client.? 172.21.15.86:0/2508859297' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T22:22:38.505 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:38 smithi067 ceph-mon[21875]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 117 KiB/s, 0 objects/s recovering 2024-04-22T22:22:38.505 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:38 smithi067 ceph-mon[21875]: from='client.? 172.21.15.86:0/2201254328' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dc347749-843b-4fa9-9ee5-d4fd966e3c41"}]: dispatch 2024-04-22T22:22:38.505 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:38 smithi067 ceph-mon[21875]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dc347749-843b-4fa9-9ee5-d4fd966e3c41"}]: dispatch 2024-04-22T22:22:38.505 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:38 smithi067 ceph-mon[21875]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "dc347749-843b-4fa9-9ee5-d4fd966e3c41"}]': finished 2024-04-22T22:22:38.505 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:38 smithi067 ceph-mon[21875]: osdmap e52: 8 total, 7 up, 8 in 2024-04-22T22:22:38.505 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:38 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T22:22:38.505 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:38 smithi067 ceph-mon[21875]: from='client.? 172.21.15.86:0/2508859297' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T22:22:38.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:38 smithi067 ceph-mon[26633]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 117 KiB/s, 0 objects/s recovering 2024-04-22T22:22:38.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:38 smithi067 ceph-mon[26633]: from='client.? 172.21.15.86:0/2201254328' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dc347749-843b-4fa9-9ee5-d4fd966e3c41"}]: dispatch 2024-04-22T22:22:38.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:38 smithi067 ceph-mon[26633]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dc347749-843b-4fa9-9ee5-d4fd966e3c41"}]: dispatch 2024-04-22T22:22:38.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:38 smithi067 ceph-mon[26633]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "dc347749-843b-4fa9-9ee5-d4fd966e3c41"}]': finished 2024-04-22T22:22:38.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:38 smithi067 ceph-mon[26633]: osdmap e52: 8 total, 7 up, 8 in 2024-04-22T22:22:38.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:38 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T22:22:38.506 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:38 smithi067 ceph-mon[26633]: from='client.? 172.21.15.86:0/2508859297' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-22T22:22:40.504 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:40 smithi086 ceph-mon[22689]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-22T22:22:40.505 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:40 smithi067 ceph-mon[21875]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-22T22:22:40.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:40 smithi067 ceph-mon[26633]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-22T22:22:42.118 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:42 smithi086 ceph-mon[22689]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-04-22T22:22:42.505 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:42 smithi067 ceph-mon[21875]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-04-22T22:22:42.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:42 smithi067 ceph-mon[26633]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-04-22T22:22:44.149 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:44 smithi086 ceph-mon[22689]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-22T22:22:44.150 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:44 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-22T22:22:44.150 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:44 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:22:44.505 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:44 smithi067 ceph-mon[21875]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-22T22:22:44.505 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:44 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-22T22:22:44.505 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:44 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:22:44.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:44 smithi067 ceph-mon[26633]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-22T22:22:44.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:44 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-22T22:22:44.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:44 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:22:45.352 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:45 smithi086 ceph-mon[22689]: Deploying daemon osd.7 on smithi086 2024-04-22T22:22:45.505 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:45 smithi067 ceph-mon[21875]: Deploying daemon osd.7 on smithi086 2024-04-22T22:22:45.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:45 smithi067 ceph-mon[26633]: Deploying daemon osd.7 on smithi086 2024-04-22T22:22:46.253 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:46 smithi086 ceph-mon[22689]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 55 KiB/s, 0 objects/s recovering 2024-04-22T22:22:46.505 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:46 smithi067 ceph-mon[21875]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 55 KiB/s, 0 objects/s recovering 2024-04-22T22:22:46.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:46 smithi067 ceph-mon[26633]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 55 KiB/s, 0 objects/s recovering 2024-04-22T22:22:48.504 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:48 smithi086 ceph-mon[22689]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T22:22:48.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:48 smithi067 ceph-mon[26633]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T22:22:48.505 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:48 smithi067 ceph-mon[21875]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T22:22:49.477 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:49 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:49.477 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:49 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:22:49.478 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:49 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:22:49.478 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:49 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:22:49.478 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:49 smithi086 ceph-mon[22689]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T22:22:49.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:49 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:49.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:49 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:22:49.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:49 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:22:49.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:49 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:22:49.505 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:49 smithi067 ceph-mon[26633]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T22:22:49.505 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:49 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:49.506 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:49 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:22:49.506 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:49 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:22:49.506 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:49 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:22:49.506 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:49 smithi067 ceph-mon[21875]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T22:22:51.261 INFO:teuthology.orchestra.run.smithi086.stdout:Created osd(s) 7 on host 'smithi086' 2024-04-22T22:22:51.815 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:51 smithi086 ceph-mon[22689]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T22:22:51.815 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:51 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:51.815 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:51 smithi086 ceph-mon[22689]: from='osd.7 [v2:172.21.15.86:6824/564124875,v1:172.21.15.86:6825/564124875]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T22:22:51.815 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:51 smithi086 ceph-mon[22689]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T22:22:51.839 DEBUG:teuthology.orchestra.run.smithi086:osd.7> sudo journalctl -f -n 0 -u ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@osd.7.service 2024-04-22T22:22:51.841 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-04-22T22:22:51.842 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph osd stat -f json 2024-04-22T22:22:51.868 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:51 smithi067 ceph-mon[21875]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T22:22:51.868 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:51 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:51.869 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:51 smithi067 ceph-mon[21875]: from='osd.7 [v2:172.21.15.86:6824/564124875,v1:172.21.15.86:6825/564124875]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T22:22:51.869 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:51 smithi067 ceph-mon[21875]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T22:22:51.869 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:51 smithi067 ceph-mon[26633]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T22:22:51.869 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:51 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:51.869 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:51 smithi067 ceph-mon[26633]: from='osd.7 [v2:172.21.15.86:6824/564124875,v1:172.21.15.86:6825/564124875]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T22:22:51.869 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:51 smithi067 ceph-mon[26633]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-22T22:22:53.277 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:53 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:53.277 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:53 smithi086 ceph-mon[22689]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-22T22:22:53.277 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:53 smithi086 ceph-mon[22689]: osdmap e53: 8 total, 7 up, 8 in 2024-04-22T22:22:53.277 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:53 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T22:22:53.277 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:53 smithi086 ceph-mon[22689]: from='osd.7 [v2:172.21.15.86:6824/564124875,v1:172.21.15.86:6825/564124875]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-22T22:22:53.277 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:53 smithi086 ceph-mon[22689]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-22T22:22:53.277 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:53 smithi086 ceph-mon[22689]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T22:22:53.362 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:53 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:53.362 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:53 smithi067 ceph-mon[26633]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-22T22:22:53.362 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:53 smithi067 ceph-mon[26633]: osdmap e53: 8 total, 7 up, 8 in 2024-04-22T22:22:53.362 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:53 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T22:22:53.363 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:53 smithi067 ceph-mon[26633]: from='osd.7 [v2:172.21.15.86:6824/564124875,v1:172.21.15.86:6825/564124875]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-22T22:22:53.363 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:53 smithi067 ceph-mon[26633]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-22T22:22:53.363 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:53 smithi067 ceph-mon[26633]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T22:22:53.363 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:53 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:53.363 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:53 smithi067 ceph-mon[21875]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-22T22:22:53.363 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:53 smithi067 ceph-mon[21875]: osdmap e53: 8 total, 7 up, 8 in 2024-04-22T22:22:53.363 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:53 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T22:22:53.363 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:53 smithi067 ceph-mon[21875]: from='osd.7 [v2:172.21.15.86:6824/564124875,v1:172.21.15.86:6825/564124875]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-22T22:22:53.363 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:53 smithi067 ceph-mon[21875]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]: dispatch 2024-04-22T22:22:53.363 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:53 smithi067 ceph-mon[21875]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-22T22:22:53.590 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:22:53 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:22:53.288+0000 7f2c96834700 -1 osd.7 0 waiting for initial osdmap 2024-04-22T22:22:53.591 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:22:53 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:22:53.296+0000 7f2c931cf700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-22T22:22:53.875 INFO:teuthology.orchestra.run.smithi067.stdout: 2024-04-22T22:22:54.480 INFO:teuthology.orchestra.run.smithi067.stdout:{"epoch":54,"num_osds":8,"num_up_osds":7,"osd_up_since":1713824551,"num_in_osds":8,"osd_in_since":1713824557,"num_remapped_pgs":0} 2024-04-22T22:22:54.488 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:54 smithi067 ceph-mon[21875]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]': finished 2024-04-22T22:22:54.488 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:54 smithi067 ceph-mon[21875]: osdmap e54: 8 total, 7 up, 8 in 2024-04-22T22:22:54.488 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:54 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T22:22:54.488 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:54 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T22:22:54.488 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:54 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/3530729437' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-22T22:22:54.488 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:54 smithi067 ceph-mon[26633]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]': finished 2024-04-22T22:22:54.488 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:54 smithi067 ceph-mon[26633]: osdmap e54: 8 total, 7 up, 8 in 2024-04-22T22:22:54.488 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:54 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T22:22:54.489 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:54 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T22:22:54.489 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:54 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/3530729437' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-22T22:22:54.667 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:54 smithi086 ceph-mon[22689]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi086", "root=default"]}]': finished 2024-04-22T22:22:54.667 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:54 smithi086 ceph-mon[22689]: osdmap e54: 8 total, 7 up, 8 in 2024-04-22T22:22:54.667 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:54 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T22:22:54.667 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:54 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T22:22:54.667 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:54 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/3530729437' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-22T22:22:55.481 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph osd stat -f json 2024-04-22T22:22:55.754 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:55 smithi086 ceph-mon[22689]: purged_snaps scrub starts 2024-04-22T22:22:55.754 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:55 smithi086 ceph-mon[22689]: purged_snaps scrub ok 2024-04-22T22:22:55.754 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:55 smithi086 ceph-mon[22689]: osd.7 [v2:172.21.15.86:6824/564124875,v1:172.21.15.86:6825/564124875] boot 2024-04-22T22:22:55.754 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:55 smithi086 ceph-mon[22689]: osdmap e55: 8 total, 8 up, 8 in 2024-04-22T22:22:55.754 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:55 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T22:22:55.754 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:55 smithi086 ceph-mon[22689]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:22:55.754 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:55 smithi086 ceph-mon[22689]: osdmap e56: 8 total, 8 up, 8 in 2024-04-22T22:22:55.755 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:55 smithi067 ceph-mon[21875]: purged_snaps scrub starts 2024-04-22T22:22:55.755 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:55 smithi067 ceph-mon[21875]: purged_snaps scrub ok 2024-04-22T22:22:55.755 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:55 smithi067 ceph-mon[21875]: osd.7 [v2:172.21.15.86:6824/564124875,v1:172.21.15.86:6825/564124875] boot 2024-04-22T22:22:55.755 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:55 smithi067 ceph-mon[21875]: osdmap e55: 8 total, 8 up, 8 in 2024-04-22T22:22:55.755 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:55 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T22:22:55.755 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:55 smithi067 ceph-mon[21875]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:22:55.755 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:55 smithi067 ceph-mon[21875]: osdmap e56: 8 total, 8 up, 8 in 2024-04-22T22:22:55.755 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:55 smithi067 ceph-mon[26633]: purged_snaps scrub starts 2024-04-22T22:22:55.756 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:55 smithi067 ceph-mon[26633]: purged_snaps scrub ok 2024-04-22T22:22:55.756 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:55 smithi067 ceph-mon[26633]: osd.7 [v2:172.21.15.86:6824/564124875,v1:172.21.15.86:6825/564124875] boot 2024-04-22T22:22:55.756 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:55 smithi067 ceph-mon[26633]: osdmap e55: 8 total, 8 up, 8 in 2024-04-22T22:22:55.756 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:55 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T22:22:55.756 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:55 smithi067 ceph-mon[26633]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:22:55.756 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:55 smithi067 ceph-mon[26633]: osdmap e56: 8 total, 8 up, 8 in 2024-04-22T22:22:56.505 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:56 smithi067 ceph-mon[21875]: Detected new or changed devices on smithi086 2024-04-22T22:22:56.506 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:56 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:56.506 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:56 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:22:56.506 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:56 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:22:56.506 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:56 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:22:56.506 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:56 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:22:56.506 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:56 smithi067 ceph-mon[21875]: Adjusting osd_memory_target on smithi086 to 4272M 2024-04-22T22:22:56.506 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:56 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:56.506 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:56 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:56.506 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:56 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:22:56.506 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:56 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:22:56.506 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:56 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:22:56.506 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:56 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:56.506 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:56 smithi067 ceph-mon[21875]: osdmap e57: 8 total, 8 up, 8 in 2024-04-22T22:22:56.507 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:56 smithi067 ceph-mon[26633]: Detected new or changed devices on smithi086 2024-04-22T22:22:56.507 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:56 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:56.507 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:56 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:22:56.507 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:56 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:22:56.507 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:56 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:22:56.507 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:56 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:22:56.507 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:56 smithi067 ceph-mon[26633]: Adjusting osd_memory_target on smithi086 to 4272M 2024-04-22T22:22:56.507 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:56 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:56.507 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:56 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:56.507 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:56 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:22:56.507 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:56 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:22:56.507 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:56 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:22:56.507 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:56 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:56.507 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:56 smithi067 ceph-mon[26633]: osdmap e57: 8 total, 8 up, 8 in 2024-04-22T22:22:56.754 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:56 smithi086 ceph-mon[22689]: Detected new or changed devices on smithi086 2024-04-22T22:22:56.754 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:56 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:56.754 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:56 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:22:56.754 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:56 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:22:56.755 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:56 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:22:56.755 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:56 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-22T22:22:56.755 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:56 smithi086 ceph-mon[22689]: Adjusting osd_memory_target on smithi086 to 4272M 2024-04-22T22:22:56.755 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:56 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:56.755 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:56 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:56.755 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:56 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:22:56.755 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:56 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:22:56.755 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:56 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:22:56.755 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:56 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:22:56.755 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:56 smithi086 ceph-mon[22689]: osdmap e57: 8 total, 8 up, 8 in 2024-04-22T22:22:57.419 INFO:teuthology.orchestra.run.smithi067.stdout: 2024-04-22T22:22:57.755 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:57 smithi067 ceph-mon[26633]: pgmap v166: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-04-22T22:22:57.756 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:57 smithi067 ceph-mon[26633]: osdmap e58: 8 total, 8 up, 8 in 2024-04-22T22:22:57.756 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:57 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/1551819122' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-22T22:22:57.756 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:57 smithi067 ceph-mon[21875]: pgmap v166: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-04-22T22:22:57.756 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:57 smithi067 ceph-mon[21875]: osdmap e58: 8 total, 8 up, 8 in 2024-04-22T22:22:57.756 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:57 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/1551819122' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-22T22:22:58.004 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:57 smithi086 ceph-mon[22689]: pgmap v166: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-04-22T22:22:58.004 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:57 smithi086 ceph-mon[22689]: osdmap e58: 8 total, 8 up, 8 in 2024-04-22T22:22:58.004 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:57 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/1551819122' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-22T22:22:58.039 INFO:teuthology.orchestra.run.smithi067.stdout:{"epoch":58,"num_osds":8,"num_up_osds":8,"osd_up_since":1713824574,"num_in_osds":8,"osd_in_since":1713824557,"num_remapped_pgs":0} 2024-04-22T22:22:58.039 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-22T22:22:58.262 INFO:teuthology.orchestra.run.smithi067.stderr:Inferring config /var/lib/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T22:22:58.682 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:58 smithi067 ceph-mon[21875]: osdmap e59: 8 total, 8 up, 8 in 2024-04-22T22:22:58.682 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:58 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:22:58.682 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:58 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:22:58.682 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:58 smithi067 ceph-mon[26633]: osdmap e59: 8 total, 8 up, 8 in 2024-04-22T22:22:58.682 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:58 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:22:58.682 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:58 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:22:58.683 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:22:58 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:22:58.397+0000 7f055b395700 -1 calc_pg_upmaps abort due to max <= 0 2024-04-22T22:22:59.004 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:58 smithi086 ceph-mon[22689]: osdmap e59: 8 total, 8 up, 8 in 2024-04-22T22:22:59.004 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:58 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:22:59.004 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:58 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:22:59.655 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:22:59 smithi067 ceph-mon[21875]: pgmap v169: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-04-22T22:22:59.655 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:22:59 smithi067 ceph-mon[26633]: pgmap v169: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-04-22T22:23:00.004 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:22:59 smithi086 ceph-mon[22689]: pgmap v169: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-04-22T22:23:00.032 INFO:teuthology.orchestra.run.smithi067.stdout: 2024-04-22T22:23:00.032 INFO:teuthology.orchestra.run.smithi067.stdout:{"epoch":59,"fsid":"1ab1a3a8-00f6-11ef-bc93-c7b262605968","created":"2024-04-22T22:18:09.296806+0000","modified":"2024-04-22T22:22:57.560606+0000","last_up_change":"2024-04-22T22:22:54.278878+0000","last_in_change":"2024-04-22T22:22:37.413673+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-22T22:21:02.881176+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"326d6f8e-d83f-4d1d-954d-0ee4e303d013","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6802","nonce":3032669792},{"type":"v1","addr":"172.21.15.67:6803","nonce":3032669792}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6804","nonce":3032669792},{"type":"v1","addr":"172.21.15.67:6805","nonce":3032669792}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6808","nonce":3032669792},{"type":"v1","addr":"172.21.15.67:6809","nonce":3032669792}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6806","nonce":3032669792},{"type":"v1","addr":"172.21.15.67:6807","nonce":3032669792}]},"public_addr":"172.21.15.67:6803/3032669792","cluster_addr":"172.21.15.67:6805/3032669792","heartbeat_back_addr":"172.21.15.67:6809/3032669792","heartbeat_front_addr":"172.21.15.67:6807/3032669792","state":["exists","up"]},{"osd":1,"uuid":"6d65e808-a2ac-4324-add8-bb0f331f6c0d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6810","nonce":740753600},{"type":"v1","addr":"172.21.15.67:6811","nonce":740753600}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6812","nonce":740753600},{"type":"v1","addr":"172.21.15.67:6813","nonce":740753600}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6816","nonce":740753600},{"type":"v1","addr":"172.21.15.67:6817","nonce":740753600}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6814","nonce":740753600},{"type":"v1","addr":"172.21.15.67:6815","nonce":740753600}]},"public_addr":"172.21.15.67:6811/740753600","cluster_addr":"172.21.15.67:6813/740753600","heartbeat_back_addr":"172.21.15.67:6817/740753600","heartbeat_front_addr":"172.21.15.67:6815/740753600","state":["exists","up"]},{"osd":2,"uuid":"c5c9cb79-a30b-4862-828b-7682f2ed658e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6818","nonce":1694104138},{"type":"v1","addr":"172.21.15.67:6819","nonce":1694104138}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6820","nonce":1694104138},{"type":"v1","addr":"172.21.15.67:6821","nonce":1694104138}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6824","nonce":1694104138},{"type":"v1","addr":"172.21.15.67:6825","nonce":1694104138}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6822","nonce":1694104138},{"type":"v1","addr":"172.21.15.67:6823","nonce":1694104138}]},"public_addr":"172.21.15.67:6819/1694104138","cluster_addr":"172.21.15.67:6821/1694104138","heartbeat_back_addr":"172.21.15.67:6825/1694104138","heartbeat_front_addr":"172.21.15.67:6823/1694104138","state":["exists","up"]},{"osd":3,"uuid":"90c05f07-2be9-45e6-b340-80d9fbbf6cc5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6826","nonce":2129594167},{"type":"v1","addr":"172.21.15.67:6827","nonce":2129594167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6828","nonce":2129594167},{"type":"v1","addr":"172.21.15.67:6829","nonce":2129594167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6832","nonce":2129594167},{"type":"v1","addr":"172.21.15.67:6833","nonce":2129594167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6830","nonce":2129594167},{"type":"v1","addr":"172.21.15.67:6831","nonce":2129594167}]},"public_addr":"172.21.15.67:6827/2129594167","cluster_addr":"172.21.15.67:6829/2129594167","heartbeat_back_addr":"172.21.15.67:6833/2129594167","heartbeat_front_addr":"172.21.15.67:6831/2129594167","state":["exists","up"]},{"osd":4,"uuid":"a5d712df-0b70-465e-8462-0058860492a9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":3080458560},{"type":"v1","addr":"172.21.15.86:6801","nonce":3080458560}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":3080458560},{"type":"v1","addr":"172.21.15.86:6803","nonce":3080458560}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":3080458560},{"type":"v1","addr":"172.21.15.86:6807","nonce":3080458560}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":3080458560},{"type":"v1","addr":"172.21.15.86:6805","nonce":3080458560}]},"public_addr":"172.21.15.86:6801/3080458560","cluster_addr":"172.21.15.86:6803/3080458560","heartbeat_back_addr":"172.21.15.86:6807/3080458560","heartbeat_front_addr":"172.21.15.86:6805/3080458560","state":["exists","up"]},{"osd":5,"uuid":"a7412a3c-3fb5-4632-a91a-7daf2c5b221e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":451381933},{"type":"v1","addr":"172.21.15.86:6809","nonce":451381933}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6810","nonce":451381933},{"type":"v1","addr":"172.21.15.86:6811","nonce":451381933}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6814","nonce":451381933},{"type":"v1","addr":"172.21.15.86:6815","nonce":451381933}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":451381933},{"type":"v1","addr":"172.21.15.86:6813","nonce":451381933}]},"public_addr":"172.21.15.86:6809/451381933","cluster_addr":"172.21.15.86:6811/451381933","heartbeat_back_addr":"172.21.15.86:6815/451381933","heartbeat_front_addr":"172.21.15.86:6813/451381933","state":["exists","up"]},{"osd":6,"uuid":"d54fd043-fcec-437d-bd01-a07f71f97430","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2900445442},{"type":"v1","addr":"172.21.15.86:6817","nonce":2900445442}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6818","nonce":2900445442},{"type":"v1","addr":"172.21.15.86:6819","nonce":2900445442}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6822","nonce":2900445442},{"type":"v1","addr":"172.21.15.86:6823","nonce":2900445442}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2900445442},{"type":"v1","addr":"172.21.15.86:6821","nonce":2900445442}]},"public_addr":"172.21.15.86:6817/2900445442","cluster_addr":"172.21.15.86:6819/2900445442","heartbeat_back_addr":"172.21.15.86:6823/2900445442","heartbeat_front_addr":"172.21.15.86:6821/2900445442","state":["exists","up"]},{"osd":7,"uuid":"dc347749-843b-4fa9-9ee5-d4fd966e3c41","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":564124875},{"type":"v1","addr":"172.21.15.86:6825","nonce":564124875}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":564124875},{"type":"v1","addr":"172.21.15.86:6827","nonce":564124875}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":564124875},{"type":"v1","addr":"172.21.15.86:6831","nonce":564124875}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":564124875},{"type":"v1","addr":"172.21.15.86:6829","nonce":564124875}]},"public_addr":"172.21.15.86:6825/564124875","cluster_addr":"172.21.15.86:6827/564124875","heartbeat_back_addr":"172.21.15.86:6831/564124875","heartbeat_front_addr":"172.21.15.86:6829/564124875","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T22:20:13.157814+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T22:20:35.896002+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T22:20:59.416284+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T22:21:22.585656+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T22:21:43.713948+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T22:22:06.559183+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T22:22:29.118249+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T22:22:52.281236+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.67:0/910784755":"2024-04-23T22:18:58.327791+0000","172.21.15.67:0/1267963091":"2024-04-23T22:18:58.327791+0000","172.21.15.67:0/4134751820":"2024-04-23T22:18:58.327791+0000","172.21.15.67:6801/4250787876":"2024-04-23T22:18:58.327791+0000","172.21.15.67:0/2868224087":"2024-04-23T22:18:35.830991+0000","172.21.15.67:0/3520310390":"2024-04-23T22:18:35.830991+0000","172.21.15.67:6800/1447658117":"2024-04-23T22:18:35.830991+0000","172.21.15.67:0/204413874":"2024-04-23T22:18:35.830991+0000","172.21.15.67:6800/4250787876":"2024-04-23T22:18:58.327791+0000","172.21.15.67:6801/1447658117":"2024-04-23T22:18:35.830991+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-22T22:23:00.694 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-22T22:21:02.881176+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-04-22T22:23:00.694 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-04-22T22:23:00.915 INFO:teuthology.orchestra.run.smithi067.stderr:Inferring config /var/lib/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T22:23:01.004 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:00 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/3803981475' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-22T22:23:01.005 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:00 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/3803981475' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-22T22:23:01.005 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:00 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/3803981475' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-22T22:23:02.004 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:01 smithi086 ceph-mon[22689]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 88 KiB/s, 0 objects/s recovering 2024-04-22T22:23:02.005 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:01 smithi067 ceph-mon[26633]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 88 KiB/s, 0 objects/s recovering 2024-04-22T22:23:02.005 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:01 smithi067 ceph-mon[21875]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 88 KiB/s, 0 objects/s recovering 2024-04-22T22:23:02.785 INFO:teuthology.orchestra.run.smithi067.stdout:pg_num: 1 2024-04-22T22:23:03.348 INFO:tasks.cephadm:Adding prometheus.a on smithi086 2024-04-22T22:23:03.348 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph orch apply prometheus '1;smithi086=a' 2024-04-22T22:23:03.661 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:03 smithi086 ceph-mon[22689]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-04-22T22:23:03.661 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:03 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/2046681762' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-22T22:23:04.005 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:03 smithi067 ceph-mon[21875]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-04-22T22:23:04.005 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:03 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/2046681762' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-22T22:23:04.006 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:03 smithi067 ceph-mon[26633]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-04-22T22:23:04.006 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:03 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/2046681762' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-22T22:23:05.214 INFO:teuthology.orchestra.run.smithi086.stdout:Scheduled prometheus update... 2024-04-22T22:23:05.798 DEBUG:teuthology.orchestra.run.smithi086:prometheus.a> sudo journalctl -f -n 0 -u ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@prometheus.a.service 2024-04-22T22:23:05.800 INFO:tasks.cephadm:Adding node-exporter.a on smithi067 2024-04-22T22:23:05.800 INFO:tasks.cephadm:Adding node-exporter.b on smithi086 2024-04-22T22:23:05.801 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph orch apply node-exporter '2;smithi067=a;smithi086=b' 2024-04-22T22:23:05.861 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:05 smithi086 ceph-mon[22689]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:05.861 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:05 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:23:05.861 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:05 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:23:05.861 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:05 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:23:05.861 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:05 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:23:05.861 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:05 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:23:05.861 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:05 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-22T22:23:06.006 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:05 smithi067 ceph-mon[21875]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:06.006 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:05 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:23:06.006 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:05 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:23:06.006 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:05 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:23:06.006 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:05 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:23:06.006 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:05 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:23:06.006 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:05 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-22T22:23:06.006 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:05 smithi067 ceph-mon[26633]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:06.007 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:05 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:23:06.007 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:05 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:23:06.007 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:05 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:23:06.007 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:05 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:23:06.007 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:05 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' 2024-04-22T22:23:06.007 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:05 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-22T22:23:06.396 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:23:06 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ignoring --setuser ceph since I am not root 2024-04-22T22:23:06.396 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:23:06 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ignoring --setgroup ceph since I am not root 2024-04-22T22:23:06.396 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:23:06 smithi086 ceph-mgr[23764]: -- 172.21.15.86:0/3681341402 <== mon.2 v2:172.21.15.86:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55bbbb2f8340 con 0x55bbbb2d0c00 2024-04-22T22:23:06.543 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:23:06 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ignoring --setuser ceph since I am not root 2024-04-22T22:23:06.543 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:23:06 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ignoring --setgroup ceph since I am not root 2024-04-22T22:23:06.543 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:23:06 smithi067 ceph-mgr[22106]: -- 172.21.15.67:0/341479391 <== mon.1 v2:172.21.15.67:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55a8512fa340 con 0x55a8512d2800 2024-04-22T22:23:06.544 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:23:06 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:23:06.420+0000 7fe151800000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-22T22:23:06.544 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:23:06 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:23:06.553+0000 7fe151800000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-22T22:23:06.754 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:06 smithi086 ceph-mon[22689]: from='client.24338 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi086=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:23:06.754 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:06 smithi086 ceph-mon[22689]: Saving service prometheus spec with placement smithi086=a;count:1 2024-04-22T22:23:06.755 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:06 smithi086 ceph-mon[22689]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-22T22:23:06.755 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:06 smithi086 ceph-mon[22689]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-22T22:23:06.755 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:23:06 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:23:06.408+0000 7f6edb6ed000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-22T22:23:06.755 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:23:06 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:23:06.539+0000 7f6edb6ed000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-22T22:23:06.755 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:23:06 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:23:06.638+0000 7f6edb6ed000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-22T22:23:06.965 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:23:06 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:23:06.657+0000 7fe151800000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-22T22:23:06.966 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:06 smithi067 ceph-mon[26633]: from='client.24338 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi086=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:23:06.966 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:06 smithi067 ceph-mon[26633]: Saving service prometheus spec with placement smithi086=a;count:1 2024-04-22T22:23:06.966 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:06 smithi067 ceph-mon[26633]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-22T22:23:06.966 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:06 smithi067 ceph-mon[26633]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-22T22:23:06.966 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:06 smithi067 ceph-mon[21875]: from='client.24338 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi086=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:23:06.967 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:06 smithi067 ceph-mon[21875]: Saving service prometheus spec with placement smithi086=a;count:1 2024-04-22T22:23:06.967 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:06 smithi067 ceph-mon[21875]: from='mgr.14152 172.21.15.67:0/595045676' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-22T22:23:06.967 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:06 smithi067 ceph-mon[21875]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-22T22:23:07.121 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:23:06 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:23:06.947+0000 7f6edb6ed000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-22T22:23:07.121 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:23:07 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:23:07.028+0000 7f6edb6ed000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-22T22:23:07.255 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:23:06 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:23:06.976+0000 7fe151800000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-22T22:23:07.255 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:23:07 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:23:07.063+0000 7fe151800000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-22T22:23:08.495 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:23:08 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:23:08.248+0000 7fe151800000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-22T22:23:08.495 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:23:08 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:23:08.420+0000 7fe151800000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-22T22:23:08.504 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:23:08 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:23:08.161+0000 7f6edb6ed000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-22T22:23:08.505 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:23:08 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:23:08.325+0000 7f6edb6ed000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-22T22:23:08.505 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:23:08 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:23:08.406+0000 7f6edb6ed000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-22T22:23:08.755 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:23:08 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:23:08.506+0000 7fe151800000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-22T22:23:09.004 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:23:08 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:23:08.715+0000 7f6edb6ed000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-22T22:23:09.004 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:23:08 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:23:08.805+0000 7f6edb6ed000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-22T22:23:09.005 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:23:08 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:23:08.897+0000 7f6edb6ed000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-22T22:23:09.255 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:23:08 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:23:08.833+0000 7fe151800000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-22T22:23:09.256 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:23:08 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:23:08.928+0000 7fe151800000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-22T22:23:09.256 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:23:09 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:23:09.025+0000 7fe151800000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-22T22:23:09.504 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:23:09 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:23:09.134+0000 7f6edb6ed000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-22T22:23:09.730 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:23:09 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:23:09.276+0000 7fe151800000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-22T22:23:10.004 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:23:09 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:23:09.573+0000 7f6edb6ed000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-22T22:23:10.005 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:23:09 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:23:09.741+0000 7fe151800000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-22T22:23:10.430 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:23:10 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:23:10.056+0000 7f6edb6ed000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-22T22:23:10.430 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:23:10 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:23:10.148+0000 7f6edb6ed000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-22T22:23:10.430 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:23:10 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:23:10.441+0000 7f6edb6ed000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-22T22:23:10.505 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:23:10 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:23:10.247+0000 7fe151800000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-22T22:23:10.506 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:23:10 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:23:10.345+0000 7fe151800000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-22T22:23:10.754 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:23:10 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:23:10.530+0000 7f6edb6ed000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-22T22:23:11.005 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:23:10 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:23:10.655+0000 7fe151800000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-22T22:23:11.006 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:23:10 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:23:10.749+0000 7fe151800000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-22T22:23:11.253 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:23:10 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:23:10.928+0000 7f6edb6ed000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-22T22:23:11.253 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:23:11 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:23:11.060+0000 7f6edb6ed000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-22T22:23:11.253 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:23:11 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:23:11.152+0000 7f6edb6ed000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-22T22:23:11.253 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:23:11 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:23:11.265+0000 7f6edb6ed000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-22T22:23:11.504 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:23:11 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:23:11.352+0000 7f6edb6ed000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-22T22:23:11.505 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:23:11 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:23:11.170+0000 7fe151800000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-22T22:23:11.505 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:23:11 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:23:11.310+0000 7fe151800000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-22T22:23:11.505 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:23:11 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:23:11.407+0000 7fe151800000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-22T22:23:12.005 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:23:11 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:23:11.527+0000 7fe151800000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-22T22:23:12.006 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:23:11 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:23:11.621+0000 7fe151800000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-22T22:23:12.238 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:23:11 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:23:11.893+0000 7f6edb6ed000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-22T22:23:12.504 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:12 smithi086 ceph-mon[22689]: Standby manager daemon x restarted 2024-04-22T22:23:12.504 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:12 smithi086 ceph-mon[22689]: Standby manager daemon x started 2024-04-22T22:23:12.505 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:12 smithi086 ceph-mon[22689]: from='mgr.? 172.21.15.86:0/1791099098' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T22:23:12.505 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:12 smithi086 ceph-mon[22689]: from='mgr.? 172.21.15.86:0/1791099098' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T22:23:12.505 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:12 smithi086 ceph-mon[22689]: from='mgr.? 172.21.15.86:0/1791099098' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T22:23:12.505 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:12 smithi086 ceph-mon[22689]: from='mgr.? 172.21.15.86:0/1791099098' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T22:23:12.505 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:23:12 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:23:12.249+0000 7f6edb6ed000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-22T22:23:12.505 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:23:12 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: [22/Apr/2024:22:23:12] ENGINE Bus STARTING 2024-04-22T22:23:12.505 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:23:12 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: CherryPy Checker: 2024-04-22T22:23:12.505 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:23:12 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: The Application mounted at '' has an empty config. 2024-04-22T22:23:12.505 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:23:12 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: 2024-04-22T22:23:12.505 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:23:12 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: [22/Apr/2024:22:23:12] ENGINE Serving on http://:::9283 2024-04-22T22:23:12.505 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:23:12 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: [22/Apr/2024:22:23:12] ENGINE Bus STARTED 2024-04-22T22:23:12.506 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:23:12 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:23:12.201+0000 7fe151800000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-22T22:23:12.506 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:12 smithi067 ceph-mon[26633]: Standby manager daemon x restarted 2024-04-22T22:23:12.506 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:12 smithi067 ceph-mon[26633]: Standby manager daemon x started 2024-04-22T22:23:12.506 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:12 smithi067 ceph-mon[26633]: from='mgr.? 172.21.15.86:0/1791099098' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T22:23:12.506 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:12 smithi067 ceph-mon[26633]: from='mgr.? 172.21.15.86:0/1791099098' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T22:23:12.506 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:12 smithi067 ceph-mon[26633]: from='mgr.? 172.21.15.86:0/1791099098' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T22:23:12.506 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:12 smithi067 ceph-mon[26633]: from='mgr.? 172.21.15.86:0/1791099098' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T22:23:12.506 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:12 smithi067 ceph-mon[21875]: Standby manager daemon x restarted 2024-04-22T22:23:12.507 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:12 smithi067 ceph-mon[21875]: Standby manager daemon x started 2024-04-22T22:23:12.507 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:12 smithi067 ceph-mon[21875]: from='mgr.? 172.21.15.86:0/1791099098' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-22T22:23:12.507 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:12 smithi067 ceph-mon[21875]: from='mgr.? 172.21.15.86:0/1791099098' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-22T22:23:12.507 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:12 smithi067 ceph-mon[21875]: from='mgr.? 172.21.15.86:0/1791099098' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-22T22:23:12.507 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:12 smithi067 ceph-mon[21875]: from='mgr.? 172.21.15.86:0/1791099098' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-22T22:23:13.006 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:23:12 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:23:12.578+0000 7fe151800000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-22T22:23:13.560 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:13 smithi067 ceph-mon[21875]: mgrmap e17: y(active, since 4m), standbys: x 2024-04-22T22:23:13.560 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:13 smithi067 ceph-mon[21875]: Active manager daemon y restarted 2024-04-22T22:23:13.560 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:13 smithi067 ceph-mon[21875]: Activating manager daemon y 2024-04-22T22:23:13.560 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:13 smithi067 ceph-mon[21875]: osdmap e60: 8 total, 8 up, 8 in 2024-04-22T22:23:13.560 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:13 smithi067 ceph-mon[26633]: mgrmap e17: y(active, since 4m), standbys: x 2024-04-22T22:23:13.561 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:13 smithi067 ceph-mon[26633]: Active manager daemon y restarted 2024-04-22T22:23:13.561 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:13 smithi067 ceph-mon[26633]: Activating manager daemon y 2024-04-22T22:23:13.561 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:13 smithi067 ceph-mon[26633]: osdmap e60: 8 total, 8 up, 8 in 2024-04-22T22:23:13.561 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:23:13 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: [22/Apr/2024:22:23:13] ENGINE Bus STARTING 2024-04-22T22:23:13.561 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:23:13 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: CherryPy Checker: 2024-04-22T22:23:13.561 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:23:13 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: The Application mounted at '' has an empty config. 2024-04-22T22:23:13.561 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:23:13 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:23:13.561 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:23:13 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: [22/Apr/2024:22:23:13] ENGINE Serving on http://:::9283 2024-04-22T22:23:13.561 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:23:13 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: [22/Apr/2024:22:23:13] ENGINE Bus STARTED 2024-04-22T22:23:13.642 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:13 smithi086 ceph-mon[22689]: mgrmap e17: y(active, since 4m), standbys: x 2024-04-22T22:23:13.642 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:13 smithi086 ceph-mon[22689]: Active manager daemon y restarted 2024-04-22T22:23:13.642 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:13 smithi086 ceph-mon[22689]: Activating manager daemon y 2024-04-22T22:23:13.642 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:13 smithi086 ceph-mon[22689]: osdmap e60: 8 total, 8 up, 8 in 2024-04-22T22:23:14.337 INFO:teuthology.orchestra.run.smithi086.stdout:Scheduled node-exporter update... 2024-04-22T22:23:14.412 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: [22/Apr/2024:22:23:14] ENGINE Bus STARTING 2024-04-22T22:23:14.412 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[26633]: mgrmap e18: y(active, starting, since 0.745166s), standbys: x 2024-04-22T22:23:14.412 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T22:23:14.412 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T22:23:14.412 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T22:23:14.413 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T22:23:14.413 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T22:23:14.413 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T22:23:14.413 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T22:23:14.413 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T22:23:14.413 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T22:23:14.413 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T22:23:14.413 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T22:23:14.413 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T22:23:14.413 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T22:23:14.413 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T22:23:14.413 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T22:23:14.413 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T22:23:14.413 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[26633]: Manager daemon y is now available 2024-04-22T22:23:14.413 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:23:14.414 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:23:14.414 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:23:14.414 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:23:14.414 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:14.414 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:23:14.414 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:23:14.414 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[21875]: mgrmap e18: y(active, starting, since 0.745166s), standbys: x 2024-04-22T22:23:14.414 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T22:23:14.414 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T22:23:14.414 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T22:23:14.414 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T22:23:14.414 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T22:23:14.414 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T22:23:14.415 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T22:23:14.415 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T22:23:14.415 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T22:23:14.415 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T22:23:14.415 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T22:23:14.415 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T22:23:14.415 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T22:23:14.415 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T22:23:14.415 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T22:23:14.415 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T22:23:14.415 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[21875]: Manager daemon y is now available 2024-04-22T22:23:14.415 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:23:14.415 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:23:14.415 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:23:14.415 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:23:14.415 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:14.416 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:23:14.416 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:23:14.505 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:14 smithi086 ceph-mon[22689]: mgrmap e18: y(active, starting, since 0.745166s), standbys: x 2024-04-22T22:23:14.505 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:14 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-22T22:23:14.505 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:14 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-22T22:23:14.505 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:14 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-22T22:23:14.505 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:14 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-22T22:23:14.505 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:14 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-22T22:23:14.505 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:14 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-22T22:23:14.505 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:14 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-22T22:23:14.505 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:14 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-22T22:23:14.505 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:14 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-22T22:23:14.505 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:14 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-22T22:23:14.505 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:14 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-22T22:23:14.505 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:14 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-22T22:23:14.505 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:14 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-22T22:23:14.505 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:14 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-22T22:23:14.506 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:14 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-22T22:23:14.506 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:14 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-22T22:23:14.506 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:14 smithi086 ceph-mon[22689]: Manager daemon y is now available 2024-04-22T22:23:14.506 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:14 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:23:14.506 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:14 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:23:14.506 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:14 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:23:14.506 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:14 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:23:14.506 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:14 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:14.506 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:14 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:23:14.506 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:14 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:23:14.756 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: [22/Apr/2024:22:23:14] ENGINE Serving on https://172.21.15.67:7150 2024-04-22T22:23:14.756 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:23:14 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: [22/Apr/2024:22:23:14] ENGINE Bus STARTED 2024-04-22T22:23:15.072 DEBUG:teuthology.orchestra.run.smithi067:node-exporter.a> sudo journalctl -f -n 0 -u ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@node-exporter.a.service 2024-04-22T22:23:15.075 DEBUG:teuthology.orchestra.run.smithi086:node-exporter.b> sudo journalctl -f -n 0 -u ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@node-exporter.b.service 2024-04-22T22:23:15.114 INFO:tasks.cephadm:Adding alertmanager.a on smithi067 2024-04-22T22:23:15.114 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph orch apply alertmanager '1;smithi067=a' 2024-04-22T22:23:15.364 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:15 smithi067 ceph-mon[21875]: [22/Apr/2024:22:23:14] ENGINE Bus STARTING 2024-04-22T22:23:15.364 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:15 smithi067 ceph-mon[21875]: mgrmap e19: y(active, since 1.75403s), standbys: x 2024-04-22T22:23:15.365 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:15 smithi067 ceph-mon[21875]: from='client.24349 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi067=a;smithi086=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:23:15.365 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:15 smithi067 ceph-mon[21875]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:15.365 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:15 smithi067 ceph-mon[21875]: Saving service node-exporter spec with placement smithi067=a;smithi086=b;count:2 2024-04-22T22:23:15.365 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:15 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:15.365 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:15 smithi067 ceph-mon[21875]: [22/Apr/2024:22:23:14] ENGINE Serving on https://172.21.15.67:7150 2024-04-22T22:23:15.365 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:15 smithi067 ceph-mon[21875]: [22/Apr/2024:22:23:14] ENGINE Bus STARTED 2024-04-22T22:23:15.365 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:15 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:15.365 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:15 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:15.365 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:15 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:15.365 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:15 smithi067 ceph-mon[26633]: [22/Apr/2024:22:23:14] ENGINE Bus STARTING 2024-04-22T22:23:15.365 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:15 smithi067 ceph-mon[26633]: mgrmap e19: y(active, since 1.75403s), standbys: x 2024-04-22T22:23:15.366 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:15 smithi067 ceph-mon[26633]: from='client.24349 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi067=a;smithi086=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:23:15.366 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:15 smithi067 ceph-mon[26633]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:15.366 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:15 smithi067 ceph-mon[26633]: Saving service node-exporter spec with placement smithi067=a;smithi086=b;count:2 2024-04-22T22:23:15.366 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:15 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:15.366 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:15 smithi067 ceph-mon[26633]: [22/Apr/2024:22:23:14] ENGINE Serving on https://172.21.15.67:7150 2024-04-22T22:23:15.366 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:15 smithi067 ceph-mon[26633]: [22/Apr/2024:22:23:14] ENGINE Bus STARTED 2024-04-22T22:23:15.366 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:15 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:15.366 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:15 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:15.366 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:15 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:15.616 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:15 smithi086 ceph-mon[22689]: [22/Apr/2024:22:23:14] ENGINE Bus STARTING 2024-04-22T22:23:15.616 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:15 smithi086 ceph-mon[22689]: mgrmap e19: y(active, since 1.75403s), standbys: x 2024-04-22T22:23:15.617 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:15 smithi086 ceph-mon[22689]: from='client.24349 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi067=a;smithi086=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:23:15.617 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:15 smithi086 ceph-mon[22689]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:15.617 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:15 smithi086 ceph-mon[22689]: Saving service node-exporter spec with placement smithi067=a;smithi086=b;count:2 2024-04-22T22:23:15.617 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:15 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:15.617 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:15 smithi086 ceph-mon[22689]: [22/Apr/2024:22:23:14] ENGINE Serving on https://172.21.15.67:7150 2024-04-22T22:23:15.617 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:15 smithi086 ceph-mon[22689]: [22/Apr/2024:22:23:14] ENGINE Bus STARTED 2024-04-22T22:23:15.617 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:15 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:15.617 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:15 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:15.617 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:15 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:16.526 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:16 smithi086 ceph-mon[22689]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:16.527 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:16 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:16.527 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:16 smithi086 ceph-mon[22689]: Updating smithi067:/etc/ceph/ceph.conf 2024-04-22T22:23:16.527 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:16 smithi086 ceph-mon[22689]: Updating smithi067:/etc/ceph/ceph.client.admin.keyring 2024-04-22T22:23:16.527 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:16 smithi086 ceph-mon[22689]: mgrmap e20: y(active, since 2s), standbys: x 2024-04-22T22:23:16.527 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:16 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:16.527 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:16 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:16.527 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:16 smithi086 ceph-mon[22689]: Updating smithi086:/etc/ceph/ceph.conf 2024-04-22T22:23:16.527 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:16 smithi086 ceph-mon[22689]: Updating smithi086:/etc/ceph/ceph.client.admin.keyring 2024-04-22T22:23:16.527 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:16 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:16.527 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:16 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:16.756 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:16 smithi067 ceph-mon[21875]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:16.756 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:16 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:16.756 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:16 smithi067 ceph-mon[21875]: Updating smithi067:/etc/ceph/ceph.conf 2024-04-22T22:23:16.756 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:16 smithi067 ceph-mon[21875]: Updating smithi067:/etc/ceph/ceph.client.admin.keyring 2024-04-22T22:23:16.756 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:16 smithi067 ceph-mon[21875]: mgrmap e20: y(active, since 2s), standbys: x 2024-04-22T22:23:16.756 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:16 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:16.756 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:16 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:16.756 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:16 smithi067 ceph-mon[21875]: Updating smithi086:/etc/ceph/ceph.conf 2024-04-22T22:23:16.756 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:16 smithi067 ceph-mon[21875]: Updating smithi086:/etc/ceph/ceph.client.admin.keyring 2024-04-22T22:23:16.756 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:16 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:16.757 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:16 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:16.757 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:16 smithi067 ceph-mon[26633]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:16.757 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:16 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:16.757 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:16 smithi067 ceph-mon[26633]: Updating smithi067:/etc/ceph/ceph.conf 2024-04-22T22:23:16.757 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:16 smithi067 ceph-mon[26633]: Updating smithi067:/etc/ceph/ceph.client.admin.keyring 2024-04-22T22:23:16.757 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:16 smithi067 ceph-mon[26633]: mgrmap e20: y(active, since 2s), standbys: x 2024-04-22T22:23:16.757 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:16 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:16.757 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:16 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:16.757 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:16 smithi067 ceph-mon[26633]: Updating smithi086:/etc/ceph/ceph.conf 2024-04-22T22:23:16.757 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:16 smithi067 ceph-mon[26633]: Updating smithi086:/etc/ceph/ceph.client.admin.keyring 2024-04-22T22:23:16.757 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:16 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:16.757 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:16 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:17.166 INFO:teuthology.orchestra.run.smithi086.stdout:Scheduled alertmanager update... 2024-04-22T22:23:17.827 DEBUG:teuthology.orchestra.run.smithi067:alertmanager.a> sudo journalctl -f -n 0 -u ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@alertmanager.a.service 2024-04-22T22:23:17.830 INFO:tasks.cephadm:Adding grafana.a on smithi086 2024-04-22T22:23:17.830 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph orch apply grafana '1;smithi086=a' 2024-04-22T22:23:18.505 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:18 smithi086 ceph-mon[22689]: Deploying daemon prometheus.a on smithi086 2024-04-22T22:23:18.505 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:18 smithi086 ceph-mon[22689]: from='client.24383 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi067=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:23:18.505 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:18 smithi086 ceph-mon[22689]: Saving service alertmanager spec with placement smithi067=a;count:1 2024-04-22T22:23:18.505 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:18 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:18.506 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:18 smithi067 ceph-mon[21875]: Deploying daemon prometheus.a on smithi086 2024-04-22T22:23:18.506 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:18 smithi067 ceph-mon[21875]: from='client.24383 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi067=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:23:18.506 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:18 smithi067 ceph-mon[21875]: Saving service alertmanager spec with placement smithi067=a;count:1 2024-04-22T22:23:18.506 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:18 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:18.506 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:18 smithi067 ceph-mon[26633]: Deploying daemon prometheus.a on smithi086 2024-04-22T22:23:18.507 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:18 smithi067 ceph-mon[26633]: from='client.24383 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi067=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:23:18.507 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:18 smithi067 ceph-mon[26633]: Saving service alertmanager spec with placement smithi067=a;count:1 2024-04-22T22:23:18.507 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:18 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:19.505 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:19 smithi086 ceph-mon[22689]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:19.506 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:19 smithi067 ceph-mon[21875]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:19.506 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:19 smithi067 ceph-mon[26633]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:20.579 INFO:teuthology.orchestra.run.smithi086.stdout:Scheduled grafana update... 2024-04-22T22:23:21.445 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:21 smithi086 ceph-mon[22689]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:21.446 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:21 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:21.470 DEBUG:teuthology.orchestra.run.smithi086:grafana.a> sudo journalctl -f -n 0 -u ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@grafana.a.service 2024-04-22T22:23:21.473 INFO:tasks.cephadm:Setting up client nodes... 2024-04-22T22:23:21.473 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-22T22:23:21.495 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:21 smithi067 ceph-mon[21875]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:21.495 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:21 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:21.495 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:21 smithi067 ceph-mon[26633]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:21.495 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:21 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:22.504 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:22 smithi086 ceph-mon[22689]: from='client.24389 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi086=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:23:22.505 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:22 smithi086 ceph-mon[22689]: Saving service grafana spec with placement smithi086=a;count:1 2024-04-22T22:23:22.506 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:22 smithi067 ceph-mon[21875]: from='client.24389 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi086=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:23:22.506 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:22 smithi067 ceph-mon[21875]: Saving service grafana spec with placement smithi086=a;count:1 2024-04-22T22:23:22.506 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:22 smithi067 ceph-mon[26633]: from='client.24389 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi086=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:23:22.506 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:22 smithi067 ceph-mon[26633]: Saving service grafana spec with placement smithi086=a;count:1 2024-04-22T22:23:23.339 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:23 smithi067 ceph-mon[21875]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:23.339 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:23 smithi067 ceph-mon[26633]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:23.504 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:23 smithi086 ceph-mon[22689]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:23.633 INFO:teuthology.orchestra.run.smithi067.stdout:[client.0] 2024-04-22T22:23:23.633 INFO:teuthology.orchestra.run.smithi067.stdout: key = AQBb4yZmd4RDJhAAg5q8LVyq0aK1H/M6GKflTQ== 2024-04-22T22:23:24.256 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:24 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/3727652569' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-22T22:23:24.256 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:24 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/3727652569' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-22T22:23:24.256 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:24 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/3727652569' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-22T22:23:24.256 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:24 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/3727652569' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-22T22:23:24.320 DEBUG:teuthology.orchestra.run.smithi067:> set -ex 2024-04-22T22:23:24.320 DEBUG:teuthology.orchestra.run.smithi067:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-04-22T22:23:24.320 DEBUG:teuthology.orchestra.run.smithi067:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-04-22T22:23:24.401 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-22T22:23:24.504 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:24 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/3727652569' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-22T22:23:24.505 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:24 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/3727652569' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-22T22:23:25.447 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:25 smithi086 ceph-mon[22689]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:25.506 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:25 smithi067 ceph-mon[21875]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:25.506 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:25 smithi067 ceph-mon[26633]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:27.495 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:27 smithi086 ceph-mon[22689]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:27.506 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:27 smithi067 ceph-mon[21875]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:27.506 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:27 smithi067 ceph-mon[26633]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:29.072 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:23:28 smithi086 systemd[1]: Starting Ceph prometheus.a for 1ab1a3a8-00f6-11ef-bc93-c7b262605968... 2024-04-22T22:23:29.327 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:29 smithi086 ceph-mon[22689]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:29.506 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:29 smithi067 ceph-mon[21875]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:29.506 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:29 smithi067 ceph-mon[26633]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:29.796 INFO:teuthology.orchestra.run.smithi086.stdout:[client.1] 2024-04-22T22:23:29.796 INFO:teuthology.orchestra.run.smithi086.stdout: key = AQBh4yZm7BLuLxAAObdyYJ8Go0Ro3MEk6ZPN8w== 2024-04-22T22:23:30.005 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:23:29 smithi086 podman[37670]: 2024-04-22 22:23:29.662867147 +0000 UTC m=+0.120873700 container create 621e529acde66aee511d27b7a7aea92a9e464b2bb0054a871c1de94cbc2e14a5 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T22:23:30.006 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:23:29 smithi086 podman[37670]: 2024-04-22 22:23:29.567800438 +0000 UTC m=+0.025807001 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-22T22:23:30.006 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:23:29 smithi086 podman[37670]: 2024-04-22 22:23:29.839986164 +0000 UTC m=+0.297992735 container init 621e529acde66aee511d27b7a7aea92a9e464b2bb0054a871c1de94cbc2e14a5 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T22:23:30.006 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:23:29 smithi086 podman[37670]: 2024-04-22 22:23:29.843062916 +0000 UTC m=+0.301069473 container start 621e529acde66aee511d27b7a7aea92a9e464b2bb0054a871c1de94cbc2e14a5 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T22:23:30.006 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:23:29 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[37687]: ts=2024-04-22T22:23:29.866Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-22T22:23:30.006 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:23:29 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[37687]: ts=2024-04-22T22:23:29.866Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-22T22:23:30.006 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:23:29 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[37687]: ts=2024-04-22T22:23:29.866Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-22T22:23:30.006 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:23:29 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[37687]: ts=2024-04-22T22:23:29.866Z caller=main.go:518 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi086 (none))" 2024-04-22T22:23:30.006 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:23:29 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[37687]: ts=2024-04-22T22:23:29.866Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-22T22:23:30.006 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:23:29 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[37687]: ts=2024-04-22T22:23:29.866Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-22T22:23:30.006 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:23:29 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[37687]: ts=2024-04-22T22:23:29.867Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-22T22:23:30.007 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:23:29 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[37687]: ts=2024-04-22T22:23:29.868Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-22T22:23:30.007 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:23:29 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[37687]: ts=2024-04-22T22:23:29.868Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-22T22:23:30.007 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:23:29 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[37687]: ts=2024-04-22T22:23:29.871Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-22T22:23:30.007 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:23:29 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[37687]: ts=2024-04-22T22:23:29.871Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.61µs 2024-04-22T22:23:30.007 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:23:29 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[37687]: ts=2024-04-22T22:23:29.871Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-22T22:23:30.007 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:23:29 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[37687]: ts=2024-04-22T22:23:29.872Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-04-22T22:23:30.007 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:23:29 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[37687]: ts=2024-04-22T22:23:29.872Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=19.546µs wal_replay_duration=299.65µs total_replay_duration=333.031µs 2024-04-22T22:23:30.007 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:23:29 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[37687]: ts=2024-04-22T22:23:29.872Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-22T22:23:30.007 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:23:29 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[37687]: ts=2024-04-22T22:23:29.872Z caller=main.go:947 level=info msg="TSDB started" 2024-04-22T22:23:30.007 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:23:29 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[37687]: ts=2024-04-22T22:23:29.872Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-22T22:23:30.007 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:23:29 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[37687]: ts=2024-04-22T22:23:29.886Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=14.208042ms db_storage=719ns remote_storage=1.461µs web_handler=332ns query_engine=534ns scrape=356.792µs scrape_sd=16.15µs notify=971ns notify_sd=2.138µs rules=13.664481ms 2024-04-22T22:23:30.008 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:23:29 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[37687]: ts=2024-04-22T22:23:29.886Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-22T22:23:30.505 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:23:30 smithi086 bash[37670]: 621e529acde66aee511d27b7a7aea92a9e464b2bb0054a871c1de94cbc2e14a5 2024-04-22T22:23:30.505 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:23:30 smithi086 systemd[1]: Started Ceph prometheus.a for 1ab1a3a8-00f6-11ef-bc93-c7b262605968. 2024-04-22T22:23:30.615 DEBUG:teuthology.orchestra.run.smithi086:> set -ex 2024-04-22T22:23:30.615 DEBUG:teuthology.orchestra.run.smithi086:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-04-22T22:23:30.616 DEBUG:teuthology.orchestra.run.smithi086:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-04-22T22:23:30.653 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-22T22:23:30.653 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-22T22:23:30.653 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph mgr dump --format=json 2024-04-22T22:23:30.898 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:30 smithi067 ceph-mon[21875]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:30.898 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:30 smithi067 ceph-mon[21875]: from='client.? 172.21.15.86:0/682983370' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-22T22:23:30.898 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:30 smithi067 ceph-mon[21875]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-22T22:23:30.898 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:30 smithi067 ceph-mon[21875]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-22T22:23:30.898 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:30 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:30.898 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:30 smithi067 ceph-mon[21875]: Deploying daemon node-exporter.a on smithi067 2024-04-22T22:23:30.898 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:30 smithi067 ceph-mon[26633]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:30.898 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:30 smithi067 ceph-mon[26633]: from='client.? 172.21.15.86:0/682983370' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-22T22:23:30.899 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:30 smithi067 ceph-mon[26633]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-22T22:23:30.899 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:30 smithi067 ceph-mon[26633]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-22T22:23:30.899 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:30 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:30.899 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:30 smithi067 ceph-mon[26633]: Deploying daemon node-exporter.a on smithi067 2024-04-22T22:23:30.902 INFO:teuthology.orchestra.run.smithi067.stderr:Inferring config /var/lib/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T22:23:31.005 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:30 smithi086 ceph-mon[22689]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:31.005 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:30 smithi086 ceph-mon[22689]: from='client.? 172.21.15.86:0/682983370' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-22T22:23:31.005 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:30 smithi086 ceph-mon[22689]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-22T22:23:31.005 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:30 smithi086 ceph-mon[22689]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-22T22:23:31.005 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:30 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:31.005 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:30 smithi086 ceph-mon[22689]: Deploying daemon node-exporter.a on smithi067 2024-04-22T22:23:32.537 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:32 smithi067 bash[41498]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2024-04-22T22:23:32.880 INFO:teuthology.orchestra.run.smithi067.stdout: 2024-04-22T22:23:32.880 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:32 smithi067 ceph-mon[21875]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:32.881 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:32 smithi067 ceph-mon[26633]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:33.005 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:32 smithi086 ceph-mon[22689]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:33.256 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:32 smithi067 bash[41498]: Getting image source signatures 2024-04-22T22:23:33.256 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:32 smithi067 bash[41498]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2024-04-22T22:23:33.256 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:32 smithi067 bash[41498]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2024-04-22T22:23:33.256 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:32 smithi067 bash[41498]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2024-04-22T22:23:33.913 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:33 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/2852012452' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-22T22:23:33.913 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:33 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:33.913 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:33 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/2852012452' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-22T22:23:33.913 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:33 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:33.935 INFO:teuthology.orchestra.run.smithi067.stdout:{"epoch":20,"active_gid":14451,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6800","nonce":3714266249},{"type":"v1","addr":"172.21.15.67:6801","nonce":3714266249}]},"active_addr":"172.21.15.67:6801/3714266249","active_change":"2024-04-22T22:23:12.582062+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":24344,"name":"x","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.67:8443/","prometheus":"http://172.21.15.67:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":60,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.67:0","nonce":3494665173}]},{"addrvec":[{"type":"v2","addr":"172.21.15.67:0","nonce":2219265811}]},{"addrvec":[{"type":"v2","addr":"172.21.15.67:0","nonce":1951105764}]},{"addrvec":[{"type":"v2","addr":"172.21.15.67:0","nonce":1503830619}]}]}} 2024-04-22T22:23:33.940 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-22T22:23:33.940 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-22T22:23:33.941 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-22T22:23:34.005 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:33 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/2852012452' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-22T22:23:34.005 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:33 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:34.662 INFO:teuthology.orchestra.run.smithi067.stderr:Inferring config /var/lib/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T22:23:34.721 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:34 smithi067 ceph-mon[21875]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:34.721 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:34 smithi067 ceph-mon[26633]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:34.998 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:34 smithi067 bash[41498]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-04-22T22:23:35.005 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:34 smithi086 ceph-mon[22689]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:35.256 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 bash[41498]: Writing manifest to image destination 2024-04-22T22:23:35.721 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 podman[41498]: 2024-04-22 22:23:35.53506879 +0000 UTC m=+3.116862114 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-04-22T22:23:36.007 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 podman[41498]: 2024-04-22 22:23:35.733219455 +0000 UTC m=+3.315012773 container create 865795666bfc7a806edaa0c6335ec7dd7a71c9b9509f8c3466a065404a8b5c7c (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T22:23:36.007 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 podman[41498]: 2024-04-22 22:23:35.944175401 +0000 UTC m=+3.525968710 container init 865795666bfc7a806edaa0c6335ec7dd7a71c9b9509f8c3466a065404a8b5c7c (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T22:23:36.007 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 podman[41498]: 2024-04-22 22:23:35.947458011 +0000 UTC m=+3.529251322 container start 865795666bfc7a806edaa0c6335ec7dd7a71c9b9509f8c3466a065404a8b5c7c (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T22:23:36.007 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.952Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-04-22T22:23:36.007 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.952Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-04-22T22:23:36.008 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.952Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-04-22T22:23:36.008 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.952Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-22T22:23:36.008 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-04-22T22:23:36.008 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=arp 2024-04-22T22:23:36.008 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=bcache 2024-04-22T22:23:36.008 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=bonding 2024-04-22T22:23:36.008 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=btrfs 2024-04-22T22:23:36.008 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=conntrack 2024-04-22T22:23:36.008 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=cpu 2024-04-22T22:23:36.008 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-04-22T22:23:36.008 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=diskstats 2024-04-22T22:23:36.008 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=dmi 2024-04-22T22:23:36.008 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=edac 2024-04-22T22:23:36.008 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=entropy 2024-04-22T22:23:36.008 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-04-22T22:23:36.008 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=filefd 2024-04-22T22:23:36.009 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=filesystem 2024-04-22T22:23:36.009 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=hwmon 2024-04-22T22:23:36.009 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=infiniband 2024-04-22T22:23:36.009 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=ipvs 2024-04-22T22:23:36.009 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=loadavg 2024-04-22T22:23:36.009 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=mdadm 2024-04-22T22:23:36.009 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=meminfo 2024-04-22T22:23:36.009 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=netclass 2024-04-22T22:23:36.009 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=netdev 2024-04-22T22:23:36.009 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=netstat 2024-04-22T22:23:36.009 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=nfs 2024-04-22T22:23:36.009 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=nfsd 2024-04-22T22:23:36.009 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=nvme 2024-04-22T22:23:36.009 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=os 2024-04-22T22:23:36.009 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-04-22T22:23:36.009 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=pressure 2024-04-22T22:23:36.009 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=rapl 2024-04-22T22:23:36.010 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=schedstat 2024-04-22T22:23:36.010 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=sockstat 2024-04-22T22:23:36.010 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=softnet 2024-04-22T22:23:36.010 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=stat 2024-04-22T22:23:36.010 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=tapestats 2024-04-22T22:23:36.010 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=textfile 2024-04-22T22:23:36.010 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-04-22T22:23:36.010 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=time 2024-04-22T22:23:36.010 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-04-22T22:23:36.010 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=uname 2024-04-22T22:23:36.010 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=vmstat 2024-04-22T22:23:36.010 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=xfs 2024-04-22T22:23:36.010 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:115 level=info collector=zfs 2024-04-22T22:23:36.010 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.953Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-04-22T22:23:36.010 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:35 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a[41672]: ts=2024-04-22T22:23:35.954Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-04-22T22:23:36.506 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:36 smithi067 bash[41498]: 865795666bfc7a806edaa0c6335ec7dd7a71c9b9509f8c3466a065404a8b5c7c 2024-04-22T22:23:36.507 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:23:36 smithi067 systemd[1]: Started Ceph node-exporter.a for 1ab1a3a8-00f6-11ef-bc93-c7b262605968. 2024-04-22T22:23:37.256 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:37 smithi086 ceph-mon[22689]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:37.256 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:37 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:37.256 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:37 smithi086 ceph-mon[22689]: Deploying daemon node-exporter.b on smithi086 2024-04-22T22:23:37.281 INFO:teuthology.orchestra.run.smithi067.stdout: 2024-04-22T22:23:37.281 INFO:teuthology.orchestra.run.smithi067.stdout:{"epoch":60,"fsid":"1ab1a3a8-00f6-11ef-bc93-c7b262605968","created":"2024-04-22T22:18:09.296806+0000","modified":"2024-04-22T22:23:12.581689+0000","last_up_change":"2024-04-22T22:22:54.278878+0000","last_in_change":"2024-04-22T22:22:37.413673+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-22T22:21:02.881176+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"326d6f8e-d83f-4d1d-954d-0ee4e303d013","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6802","nonce":3032669792},{"type":"v1","addr":"172.21.15.67:6803","nonce":3032669792}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6804","nonce":3032669792},{"type":"v1","addr":"172.21.15.67:6805","nonce":3032669792}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6808","nonce":3032669792},{"type":"v1","addr":"172.21.15.67:6809","nonce":3032669792}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6806","nonce":3032669792},{"type":"v1","addr":"172.21.15.67:6807","nonce":3032669792}]},"public_addr":"172.21.15.67:6803/3032669792","cluster_addr":"172.21.15.67:6805/3032669792","heartbeat_back_addr":"172.21.15.67:6809/3032669792","heartbeat_front_addr":"172.21.15.67:6807/3032669792","state":["exists","up"]},{"osd":1,"uuid":"6d65e808-a2ac-4324-add8-bb0f331f6c0d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6810","nonce":740753600},{"type":"v1","addr":"172.21.15.67:6811","nonce":740753600}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6812","nonce":740753600},{"type":"v1","addr":"172.21.15.67:6813","nonce":740753600}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6816","nonce":740753600},{"type":"v1","addr":"172.21.15.67:6817","nonce":740753600}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6814","nonce":740753600},{"type":"v1","addr":"172.21.15.67:6815","nonce":740753600}]},"public_addr":"172.21.15.67:6811/740753600","cluster_addr":"172.21.15.67:6813/740753600","heartbeat_back_addr":"172.21.15.67:6817/740753600","heartbeat_front_addr":"172.21.15.67:6815/740753600","state":["exists","up"]},{"osd":2,"uuid":"c5c9cb79-a30b-4862-828b-7682f2ed658e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6818","nonce":1694104138},{"type":"v1","addr":"172.21.15.67:6819","nonce":1694104138}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6820","nonce":1694104138},{"type":"v1","addr":"172.21.15.67:6821","nonce":1694104138}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6824","nonce":1694104138},{"type":"v1","addr":"172.21.15.67:6825","nonce":1694104138}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6822","nonce":1694104138},{"type":"v1","addr":"172.21.15.67:6823","nonce":1694104138}]},"public_addr":"172.21.15.67:6819/1694104138","cluster_addr":"172.21.15.67:6821/1694104138","heartbeat_back_addr":"172.21.15.67:6825/1694104138","heartbeat_front_addr":"172.21.15.67:6823/1694104138","state":["exists","up"]},{"osd":3,"uuid":"90c05f07-2be9-45e6-b340-80d9fbbf6cc5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6826","nonce":2129594167},{"type":"v1","addr":"172.21.15.67:6827","nonce":2129594167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6828","nonce":2129594167},{"type":"v1","addr":"172.21.15.67:6829","nonce":2129594167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6832","nonce":2129594167},{"type":"v1","addr":"172.21.15.67:6833","nonce":2129594167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6830","nonce":2129594167},{"type":"v1","addr":"172.21.15.67:6831","nonce":2129594167}]},"public_addr":"172.21.15.67:6827/2129594167","cluster_addr":"172.21.15.67:6829/2129594167","heartbeat_back_addr":"172.21.15.67:6833/2129594167","heartbeat_front_addr":"172.21.15.67:6831/2129594167","state":["exists","up"]},{"osd":4,"uuid":"a5d712df-0b70-465e-8462-0058860492a9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":3080458560},{"type":"v1","addr":"172.21.15.86:6801","nonce":3080458560}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":3080458560},{"type":"v1","addr":"172.21.15.86:6803","nonce":3080458560}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":3080458560},{"type":"v1","addr":"172.21.15.86:6807","nonce":3080458560}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":3080458560},{"type":"v1","addr":"172.21.15.86:6805","nonce":3080458560}]},"public_addr":"172.21.15.86:6801/3080458560","cluster_addr":"172.21.15.86:6803/3080458560","heartbeat_back_addr":"172.21.15.86:6807/3080458560","heartbeat_front_addr":"172.21.15.86:6805/3080458560","state":["exists","up"]},{"osd":5,"uuid":"a7412a3c-3fb5-4632-a91a-7daf2c5b221e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":451381933},{"type":"v1","addr":"172.21.15.86:6809","nonce":451381933}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6810","nonce":451381933},{"type":"v1","addr":"172.21.15.86:6811","nonce":451381933}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6814","nonce":451381933},{"type":"v1","addr":"172.21.15.86:6815","nonce":451381933}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":451381933},{"type":"v1","addr":"172.21.15.86:6813","nonce":451381933}]},"public_addr":"172.21.15.86:6809/451381933","cluster_addr":"172.21.15.86:6811/451381933","heartbeat_back_addr":"172.21.15.86:6815/451381933","heartbeat_front_addr":"172.21.15.86:6813/451381933","state":["exists","up"]},{"osd":6,"uuid":"d54fd043-fcec-437d-bd01-a07f71f97430","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2900445442},{"type":"v1","addr":"172.21.15.86:6817","nonce":2900445442}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6818","nonce":2900445442},{"type":"v1","addr":"172.21.15.86:6819","nonce":2900445442}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6822","nonce":2900445442},{"type":"v1","addr":"172.21.15.86:6823","nonce":2900445442}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2900445442},{"type":"v1","addr":"172.21.15.86:6821","nonce":2900445442}]},"public_addr":"172.21.15.86:6817/2900445442","cluster_addr":"172.21.15.86:6819/2900445442","heartbeat_back_addr":"172.21.15.86:6823/2900445442","heartbeat_front_addr":"172.21.15.86:6821/2900445442","state":["exists","up"]},{"osd":7,"uuid":"dc347749-843b-4fa9-9ee5-d4fd966e3c41","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":564124875},{"type":"v1","addr":"172.21.15.86:6825","nonce":564124875}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":564124875},{"type":"v1","addr":"172.21.15.86:6827","nonce":564124875}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":564124875},{"type":"v1","addr":"172.21.15.86:6831","nonce":564124875}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":564124875},{"type":"v1","addr":"172.21.15.86:6829","nonce":564124875}]},"public_addr":"172.21.15.86:6825/564124875","cluster_addr":"172.21.15.86:6827/564124875","heartbeat_back_addr":"172.21.15.86:6831/564124875","heartbeat_front_addr":"172.21.15.86:6829/564124875","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T22:20:13.157814+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T22:20:35.896002+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T22:20:59.416284+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T22:21:22.585656+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T22:21:43.713948+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T22:22:06.559183+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T22:22:29.118249+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T22:22:52.281236+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.67:0/121635657":"2024-04-23T22:23:12.581655+0000","172.21.15.67:0/3240066061":"2024-04-23T22:23:12.581655+0000","172.21.15.67:6801/288040075":"2024-04-23T22:23:12.581655+0000","172.21.15.67:0/910784755":"2024-04-23T22:18:58.327791+0000","172.21.15.67:0/1267963091":"2024-04-23T22:18:58.327791+0000","172.21.15.67:0/4134751820":"2024-04-23T22:18:58.327791+0000","172.21.15.67:6801/4250787876":"2024-04-23T22:18:58.327791+0000","172.21.15.67:0/2868224087":"2024-04-23T22:18:35.830991+0000","172.21.15.67:0/3376956307":"2024-04-23T22:23:12.581655+0000","172.21.15.67:0/3520310390":"2024-04-23T22:18:35.830991+0000","172.21.15.67:6800/1447658117":"2024-04-23T22:18:35.830991+0000","172.21.15.67:0/204413874":"2024-04-23T22:18:35.830991+0000","172.21.15.67:6800/288040075":"2024-04-23T22:23:12.581655+0000","172.21.15.67:0/2710747697":"2024-04-23T22:23:12.581655+0000","172.21.15.67:6800/4250787876":"2024-04-23T22:18:58.327791+0000","172.21.15.67:6801/1447658117":"2024-04-23T22:18:35.830991+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-22T22:23:37.507 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:37 smithi067 ceph-mon[21875]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:37.507 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:37 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:37.507 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:37 smithi067 ceph-mon[21875]: Deploying daemon node-exporter.b on smithi086 2024-04-22T22:23:37.507 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:37 smithi067 ceph-mon[26633]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:37.507 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:37 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:37.507 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:37 smithi067 ceph-mon[26633]: Deploying daemon node-exporter.b on smithi086 2024-04-22T22:23:37.755 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:37 smithi086 bash[37951]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2024-04-22T22:23:37.928 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-22T22:23:37.929 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-22T22:23:38.157 INFO:teuthology.orchestra.run.smithi067.stderr:Inferring config /var/lib/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T22:23:38.256 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:38 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/1263778770' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-22T22:23:38.257 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:38 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/1263778770' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-22T22:23:38.505 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:38 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/1263778770' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-22T22:23:39.100 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:38 smithi086 bash[37951]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-04-22T22:23:39.100 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:38 smithi086 bash[37951]: Writing manifest to image destination 2024-04-22T22:23:39.440 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:39 smithi067 ceph-mon[21875]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:39.440 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:39 smithi067 ceph-mon[26633]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:39.468 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-mon[22689]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:39.469 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 podman[37951]: 2024-04-22 22:23:39.258024824 +0000 UTC m=+2.271594095 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-04-22T22:23:39.469 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 podman[37951]: 2024-04-22 22:23:39.347387069 +0000 UTC m=+2.360956329 container create bc12658d1c9ca27b743aff7e4998c7c3f1a128c8a72903646e95eb170c0b1ba1 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T22:23:39.872 INFO:teuthology.orchestra.run.smithi067.stdout: 2024-04-22T22:23:39.873 INFO:teuthology.orchestra.run.smithi067.stdout:{"epoch":60,"fsid":"1ab1a3a8-00f6-11ef-bc93-c7b262605968","created":"2024-04-22T22:18:09.296806+0000","modified":"2024-04-22T22:23:12.581689+0000","last_up_change":"2024-04-22T22:22:54.278878+0000","last_in_change":"2024-04-22T22:22:37.413673+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-22T22:21:02.881176+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"326d6f8e-d83f-4d1d-954d-0ee4e303d013","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6802","nonce":3032669792},{"type":"v1","addr":"172.21.15.67:6803","nonce":3032669792}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6804","nonce":3032669792},{"type":"v1","addr":"172.21.15.67:6805","nonce":3032669792}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6808","nonce":3032669792},{"type":"v1","addr":"172.21.15.67:6809","nonce":3032669792}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6806","nonce":3032669792},{"type":"v1","addr":"172.21.15.67:6807","nonce":3032669792}]},"public_addr":"172.21.15.67:6803/3032669792","cluster_addr":"172.21.15.67:6805/3032669792","heartbeat_back_addr":"172.21.15.67:6809/3032669792","heartbeat_front_addr":"172.21.15.67:6807/3032669792","state":["exists","up"]},{"osd":1,"uuid":"6d65e808-a2ac-4324-add8-bb0f331f6c0d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6810","nonce":740753600},{"type":"v1","addr":"172.21.15.67:6811","nonce":740753600}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6812","nonce":740753600},{"type":"v1","addr":"172.21.15.67:6813","nonce":740753600}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6816","nonce":740753600},{"type":"v1","addr":"172.21.15.67:6817","nonce":740753600}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6814","nonce":740753600},{"type":"v1","addr":"172.21.15.67:6815","nonce":740753600}]},"public_addr":"172.21.15.67:6811/740753600","cluster_addr":"172.21.15.67:6813/740753600","heartbeat_back_addr":"172.21.15.67:6817/740753600","heartbeat_front_addr":"172.21.15.67:6815/740753600","state":["exists","up"]},{"osd":2,"uuid":"c5c9cb79-a30b-4862-828b-7682f2ed658e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6818","nonce":1694104138},{"type":"v1","addr":"172.21.15.67:6819","nonce":1694104138}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6820","nonce":1694104138},{"type":"v1","addr":"172.21.15.67:6821","nonce":1694104138}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6824","nonce":1694104138},{"type":"v1","addr":"172.21.15.67:6825","nonce":1694104138}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6822","nonce":1694104138},{"type":"v1","addr":"172.21.15.67:6823","nonce":1694104138}]},"public_addr":"172.21.15.67:6819/1694104138","cluster_addr":"172.21.15.67:6821/1694104138","heartbeat_back_addr":"172.21.15.67:6825/1694104138","heartbeat_front_addr":"172.21.15.67:6823/1694104138","state":["exists","up"]},{"osd":3,"uuid":"90c05f07-2be9-45e6-b340-80d9fbbf6cc5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6826","nonce":2129594167},{"type":"v1","addr":"172.21.15.67:6827","nonce":2129594167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6828","nonce":2129594167},{"type":"v1","addr":"172.21.15.67:6829","nonce":2129594167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6832","nonce":2129594167},{"type":"v1","addr":"172.21.15.67:6833","nonce":2129594167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.67:6830","nonce":2129594167},{"type":"v1","addr":"172.21.15.67:6831","nonce":2129594167}]},"public_addr":"172.21.15.67:6827/2129594167","cluster_addr":"172.21.15.67:6829/2129594167","heartbeat_back_addr":"172.21.15.67:6833/2129594167","heartbeat_front_addr":"172.21.15.67:6831/2129594167","state":["exists","up"]},{"osd":4,"uuid":"a5d712df-0b70-465e-8462-0058860492a9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6800","nonce":3080458560},{"type":"v1","addr":"172.21.15.86:6801","nonce":3080458560}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6802","nonce":3080458560},{"type":"v1","addr":"172.21.15.86:6803","nonce":3080458560}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6806","nonce":3080458560},{"type":"v1","addr":"172.21.15.86:6807","nonce":3080458560}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6804","nonce":3080458560},{"type":"v1","addr":"172.21.15.86:6805","nonce":3080458560}]},"public_addr":"172.21.15.86:6801/3080458560","cluster_addr":"172.21.15.86:6803/3080458560","heartbeat_back_addr":"172.21.15.86:6807/3080458560","heartbeat_front_addr":"172.21.15.86:6805/3080458560","state":["exists","up"]},{"osd":5,"uuid":"a7412a3c-3fb5-4632-a91a-7daf2c5b221e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6808","nonce":451381933},{"type":"v1","addr":"172.21.15.86:6809","nonce":451381933}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6810","nonce":451381933},{"type":"v1","addr":"172.21.15.86:6811","nonce":451381933}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6814","nonce":451381933},{"type":"v1","addr":"172.21.15.86:6815","nonce":451381933}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6812","nonce":451381933},{"type":"v1","addr":"172.21.15.86:6813","nonce":451381933}]},"public_addr":"172.21.15.86:6809/451381933","cluster_addr":"172.21.15.86:6811/451381933","heartbeat_back_addr":"172.21.15.86:6815/451381933","heartbeat_front_addr":"172.21.15.86:6813/451381933","state":["exists","up"]},{"osd":6,"uuid":"d54fd043-fcec-437d-bd01-a07f71f97430","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6816","nonce":2900445442},{"type":"v1","addr":"172.21.15.86:6817","nonce":2900445442}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6818","nonce":2900445442},{"type":"v1","addr":"172.21.15.86:6819","nonce":2900445442}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6822","nonce":2900445442},{"type":"v1","addr":"172.21.15.86:6823","nonce":2900445442}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6820","nonce":2900445442},{"type":"v1","addr":"172.21.15.86:6821","nonce":2900445442}]},"public_addr":"172.21.15.86:6817/2900445442","cluster_addr":"172.21.15.86:6819/2900445442","heartbeat_back_addr":"172.21.15.86:6823/2900445442","heartbeat_front_addr":"172.21.15.86:6821/2900445442","state":["exists","up"]},{"osd":7,"uuid":"dc347749-843b-4fa9-9ee5-d4fd966e3c41","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6824","nonce":564124875},{"type":"v1","addr":"172.21.15.86:6825","nonce":564124875}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6826","nonce":564124875},{"type":"v1","addr":"172.21.15.86:6827","nonce":564124875}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6830","nonce":564124875},{"type":"v1","addr":"172.21.15.86:6831","nonce":564124875}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.86:6828","nonce":564124875},{"type":"v1","addr":"172.21.15.86:6829","nonce":564124875}]},"public_addr":"172.21.15.86:6825/564124875","cluster_addr":"172.21.15.86:6827/564124875","heartbeat_back_addr":"172.21.15.86:6831/564124875","heartbeat_front_addr":"172.21.15.86:6829/564124875","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T22:20:13.157814+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T22:20:35.896002+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T22:20:59.416284+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T22:21:22.585656+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T22:21:43.713948+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T22:22:06.559183+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T22:22:29.118249+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-22T22:22:52.281236+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.67:0/121635657":"2024-04-23T22:23:12.581655+0000","172.21.15.67:0/3240066061":"2024-04-23T22:23:12.581655+0000","172.21.15.67:6801/288040075":"2024-04-23T22:23:12.581655+0000","172.21.15.67:0/910784755":"2024-04-23T22:18:58.327791+0000","172.21.15.67:0/1267963091":"2024-04-23T22:18:58.327791+0000","172.21.15.67:0/4134751820":"2024-04-23T22:18:58.327791+0000","172.21.15.67:6801/4250787876":"2024-04-23T22:18:58.327791+0000","172.21.15.67:0/2868224087":"2024-04-23T22:18:35.830991+0000","172.21.15.67:0/3376956307":"2024-04-23T22:23:12.581655+0000","172.21.15.67:0/3520310390":"2024-04-23T22:18:35.830991+0000","172.21.15.67:6800/1447658117":"2024-04-23T22:18:35.830991+0000","172.21.15.67:0/204413874":"2024-04-23T22:18:35.830991+0000","172.21.15.67:6800/288040075":"2024-04-23T22:23:12.581655+0000","172.21.15.67:0/2710747697":"2024-04-23T22:23:12.581655+0000","172.21.15.67:6800/4250787876":"2024-04-23T22:18:58.327791+0000","172.21.15.67:6801/1447658117":"2024-04-23T22:18:35.830991+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-22T22:23:40.006 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 podman[37951]: 2024-04-22 22:23:39.58347639 +0000 UTC m=+2.597045651 container init bc12658d1c9ca27b743aff7e4998c7c3f1a128c8a72903646e95eb170c0b1ba1 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T22:23:40.006 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 podman[37951]: 2024-04-22 22:23:39.586676972 +0000 UTC m=+2.600246233 container start bc12658d1c9ca27b743aff7e4998c7c3f1a128c8a72903646e95eb170c0b1ba1 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T22:23:40.006 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.591Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-04-22T22:23:40.006 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.591Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-04-22T22:23:40.006 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.592Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-04-22T22:23:40.007 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.592Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-22T22:23:40.007 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.592Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-04-22T22:23:40.007 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.592Z caller=node_exporter.go:115 level=info collector=arp 2024-04-22T22:23:40.007 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.592Z caller=node_exporter.go:115 level=info collector=bcache 2024-04-22T22:23:40.007 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.592Z caller=node_exporter.go:115 level=info collector=bonding 2024-04-22T22:23:40.007 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.592Z caller=node_exporter.go:115 level=info collector=btrfs 2024-04-22T22:23:40.007 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.592Z caller=node_exporter.go:115 level=info collector=conntrack 2024-04-22T22:23:40.007 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.592Z caller=node_exporter.go:115 level=info collector=cpu 2024-04-22T22:23:40.007 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.592Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-04-22T22:23:40.007 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.592Z caller=node_exporter.go:115 level=info collector=diskstats 2024-04-22T22:23:40.007 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.592Z caller=node_exporter.go:115 level=info collector=dmi 2024-04-22T22:23:40.008 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.592Z caller=node_exporter.go:115 level=info collector=edac 2024-04-22T22:23:40.008 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.592Z caller=node_exporter.go:115 level=info collector=entropy 2024-04-22T22:23:40.008 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.592Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-04-22T22:23:40.008 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.592Z caller=node_exporter.go:115 level=info collector=filefd 2024-04-22T22:23:40.008 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.593Z caller=node_exporter.go:115 level=info collector=filesystem 2024-04-22T22:23:40.008 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.593Z caller=node_exporter.go:115 level=info collector=hwmon 2024-04-22T22:23:40.009 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.593Z caller=node_exporter.go:115 level=info collector=infiniband 2024-04-22T22:23:40.009 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.593Z caller=node_exporter.go:115 level=info collector=ipvs 2024-04-22T22:23:40.009 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.593Z caller=node_exporter.go:115 level=info collector=loadavg 2024-04-22T22:23:40.009 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.593Z caller=node_exporter.go:115 level=info collector=mdadm 2024-04-22T22:23:40.009 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.593Z caller=node_exporter.go:115 level=info collector=meminfo 2024-04-22T22:23:40.009 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.593Z caller=node_exporter.go:115 level=info collector=netclass 2024-04-22T22:23:40.009 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.593Z caller=node_exporter.go:115 level=info collector=netdev 2024-04-22T22:23:40.009 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.593Z caller=node_exporter.go:115 level=info collector=netstat 2024-04-22T22:23:40.009 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.593Z caller=node_exporter.go:115 level=info collector=nfs 2024-04-22T22:23:40.009 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.593Z caller=node_exporter.go:115 level=info collector=nfsd 2024-04-22T22:23:40.009 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.593Z caller=node_exporter.go:115 level=info collector=nvme 2024-04-22T22:23:40.009 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.593Z caller=node_exporter.go:115 level=info collector=os 2024-04-22T22:23:40.010 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.593Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-04-22T22:23:40.010 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.593Z caller=node_exporter.go:115 level=info collector=pressure 2024-04-22T22:23:40.010 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.593Z caller=node_exporter.go:115 level=info collector=rapl 2024-04-22T22:23:40.010 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.593Z caller=node_exporter.go:115 level=info collector=schedstat 2024-04-22T22:23:40.010 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.593Z caller=node_exporter.go:115 level=info collector=sockstat 2024-04-22T22:23:40.010 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.593Z caller=node_exporter.go:115 level=info collector=softnet 2024-04-22T22:23:40.010 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.593Z caller=node_exporter.go:115 level=info collector=stat 2024-04-22T22:23:40.010 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.593Z caller=node_exporter.go:115 level=info collector=tapestats 2024-04-22T22:23:40.010 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.593Z caller=node_exporter.go:115 level=info collector=textfile 2024-04-22T22:23:40.010 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.593Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-04-22T22:23:40.011 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.593Z caller=node_exporter.go:115 level=info collector=time 2024-04-22T22:23:40.011 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.593Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-04-22T22:23:40.011 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.593Z caller=node_exporter.go:115 level=info collector=uname 2024-04-22T22:23:40.011 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.593Z caller=node_exporter.go:115 level=info collector=vmstat 2024-04-22T22:23:40.011 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.593Z caller=node_exporter.go:115 level=info collector=xfs 2024-04-22T22:23:40.011 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.593Z caller=node_exporter.go:115 level=info collector=zfs 2024-04-22T22:23:40.011 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.593Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-04-22T22:23:40.011 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b[38001]: ts=2024-04-22T22:23:39.593Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-04-22T22:23:40.011 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 bash[37951]: bc12658d1c9ca27b743aff7e4998c7c3f1a128c8a72903646e95eb170c0b1ba1 2024-04-22T22:23:40.011 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:23:39 smithi086 systemd[1]: Started Ceph node-exporter.b for 1ab1a3a8-00f6-11ef-bc93-c7b262605968. 2024-04-22T22:23:40.541 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-22T22:23:40.541 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-22T22:23:40.541 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-22T22:23:40.542 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-22T22:23:40.542 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-22T22:23:40.543 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-22T22:23:40.543 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-22T22:23:40.544 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-22T22:23:40.571 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:23:40 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:23:40] "GET /metrics HTTP/1.1" 200 191515 "" "Prometheus/2.33.4" 2024-04-22T22:23:40.964 INFO:teuthology.orchestra.run.smithi067.stderr:Inferring config /var/lib/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T22:23:41.003 INFO:teuthology.orchestra.run.smithi067.stderr:Inferring config /var/lib/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T22:23:41.005 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:40 smithi086 ceph-mon[22689]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:41.005 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:40 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:41.005 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:40 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:23:41.005 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:40 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:23:41.005 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:40 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:23:41.006 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:40 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/3922698520' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-22T22:23:41.006 INFO:teuthology.orchestra.run.smithi067.stderr:Inferring config /var/lib/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T22:23:41.006 INFO:teuthology.orchestra.run.smithi067.stderr:Inferring config /var/lib/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T22:23:41.007 INFO:teuthology.orchestra.run.smithi067.stderr:Inferring config /var/lib/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T22:23:41.010 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:40 smithi067 ceph-mon[21875]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:41.011 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:40 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:41.011 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:40 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:23:41.011 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:40 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:23:41.011 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:40 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:23:41.011 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:40 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/3922698520' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-22T22:23:41.011 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:40 smithi067 ceph-mon[26633]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:41.011 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:40 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:41.011 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:40 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-22T22:23:41.011 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:40 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:23:41.011 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:40 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:23:41.011 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:40 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/3922698520' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-22T22:23:41.012 INFO:teuthology.orchestra.run.smithi067.stderr:Inferring config /var/lib/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T22:23:41.012 INFO:teuthology.orchestra.run.smithi067.stderr:Inferring config /var/lib/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T22:23:41.018 INFO:teuthology.orchestra.run.smithi067.stderr:Inferring config /var/lib/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T22:23:41.755 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:23:41 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:23:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:23:43.005 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:42 smithi086 ceph-mon[22689]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:43.006 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:42 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:43.007 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:42 smithi067 ceph-mon[26633]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:43.007 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:42 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:43.007 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:42 smithi067 ceph-mon[21875]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:43.007 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:42 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:44.629 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:44 smithi067 ceph-mon[21875]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:44.630 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:44 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:44.630 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:44 smithi067 ceph-mon[26633]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:44.630 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:44 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:44.755 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:44 smithi086 ceph-mon[22689]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:44.755 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:44 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:46.757 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:46 smithi067 ceph-mon[21875]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:46.757 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:46 smithi067 ceph-mon[26633]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:47.005 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:46 smithi086 ceph-mon[22689]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:48.757 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:48 smithi067 ceph-mon[21875]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:48.757 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:48 smithi067 ceph-mon[26633]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:49.005 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:48 smithi086 ceph-mon[22689]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:50.673 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:50 smithi067 ceph-mon[21875]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:50.673 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:23:50 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:23:50] "GET /metrics HTTP/1.1" 200 191494 "" "Prometheus/2.33.4" 2024-04-22T22:23:50.673 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:50 smithi067 ceph-mon[26633]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:51.005 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:50 smithi086 ceph-mon[22689]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:51.677 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:23:51 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:23:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:23:52.147 INFO:teuthology.orchestra.run.smithi067.stdout:201863462930 2024-04-22T22:23:52.148 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-22T22:23:52.757 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:52 smithi067 ceph-mon[26633]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:52.757 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:52 smithi067 ceph-mon[21875]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:53.006 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:52 smithi086 ceph-mon[22689]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:53.129 INFO:teuthology.orchestra.run.smithi067.stdout:34359738414 2024-04-22T22:23:53.129 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-22T22:23:54.852 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:54 smithi067 ceph-mon[21875]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:54.852 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:54 smithi067 ceph-mon[26633]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:55.006 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:54 smithi086 ceph-mon[22689]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:55.189 INFO:teuthology.orchestra.run.smithi067.stdout:167503724566 2024-04-22T22:23:55.189 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-22T22:23:55.725 INFO:teuthology.orchestra.run.smithi067.stdout:133143986203 2024-04-22T22:23:55.726 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-22T22:23:56.451 INFO:teuthology.orchestra.run.smithi067.stdout:111669149728 2024-04-22T22:23:56.452 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-22T22:23:56.757 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:56 smithi067 ceph-mon[26633]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:56.757 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:56 smithi067 ceph-mon[21875]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:57.006 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:56 smithi086 ceph-mon[22689]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:57.007 INFO:teuthology.orchestra.run.smithi067.stdout:77309411364 2024-04-22T22:23:57.007 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-22T22:23:58.051 INFO:teuthology.orchestra.run.smithi067.stderr:Inferring config /var/lib/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T22:23:58.507 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:58 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:58.507 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:58 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:58.507 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:58 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:58.507 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:58 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:58.755 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:58 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:58.756 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:58 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:23:59.082 INFO:teuthology.orchestra.run.smithi067.stdout:236223201294 2024-04-22T22:23:59.083 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-22T22:23:59.415 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:59 smithi067 ceph-mon[21875]: Deploying daemon alertmanager.a on smithi067 2024-04-22T22:23:59.415 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:23:59 smithi067 ceph-mon[21875]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:59.415 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:59 smithi067 ceph-mon[26633]: Deploying daemon alertmanager.a on smithi067 2024-04-22T22:23:59.415 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:23:59 smithi067 ceph-mon[26633]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:59.423 INFO:teuthology.orchestra.run.smithi067.stderr:Inferring config /var/lib/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T22:23:59.423 INFO:teuthology.orchestra.run.smithi067.stderr:Inferring config /var/lib/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T22:23:59.423 INFO:teuthology.orchestra.run.smithi067.stderr:Inferring config /var/lib/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T22:23:59.528 INFO:teuthology.orchestra.run.smithi067.stderr:Inferring config /var/lib/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T22:23:59.529 INFO:teuthology.orchestra.run.smithi067.stderr:Inferring config /var/lib/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T22:23:59.755 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:59 smithi086 ceph-mon[22689]: Deploying daemon alertmanager.a on smithi067 2024-04-22T22:23:59.756 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:23:59 smithi086 ceph-mon[22689]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:23:59.983 INFO:teuthology.orchestra.run.smithi067.stderr:Inferring config /var/lib/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T22:24:00.869 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:24:00 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:24:00] "GET /metrics HTTP/1.1" 200 191486 "" "Prometheus/2.33.4" 2024-04-22T22:24:01.507 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:01 smithi067 ceph-mon[21875]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:01.507 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:01 smithi067 ceph-mon[26633]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:01.756 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:01 smithi086 ceph-mon[22689]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:01.756 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:24:01 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:24:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:24:01.892 INFO:teuthology.orchestra.run.smithi067.stdout:55834574890 2024-04-22T22:24:01.893 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-22T22:24:03.507 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:03 smithi067 ceph-mon[21875]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:03.507 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:03 smithi067 ceph-mon[26633]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:03.756 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:03 smithi086 ceph-mon[22689]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:04.481 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:04 smithi067 ceph-mon[21875]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:04.481 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:04 smithi067 ceph-mon[26633]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:04.756 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:04 smithi086 ceph-mon[22689]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:06.903 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:06 smithi067 ceph-mon[21875]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:06.904 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:06 smithi067 ceph-mon[26633]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:06.909 INFO:teuthology.orchestra.run.smithi067.stdout:201863462933 2024-04-22T22:24:07.006 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:06 smithi086 ceph-mon[22689]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:07.750 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:07 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/2230914156' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-22T22:24:07.750 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:07 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/2230914156' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-22T22:24:08.006 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:07 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/2230914156' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-22T22:24:08.757 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:08 smithi067 ceph-mon[26633]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:08.758 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:08 smithi067 ceph-mon[21875]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:08.834 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462930 got 201863462933 for osd.6 2024-04-22T22:24:08.834 DEBUG:teuthology.parallel:result is None 2024-04-22T22:24:09.006 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:08 smithi086 ceph-mon[22689]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:10.703 INFO:teuthology.orchestra.run.smithi067.stdout:167503724570 2024-04-22T22:24:10.822 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:10 smithi086 ceph-mon[22689]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:11.007 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:10 smithi067 ceph-mon[21875]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:11.007 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:10 smithi067 ceph-mon[26633]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:11.008 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:24:10 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:24:10] "GET /metrics HTTP/1.1" 200 191486 "" "Prometheus/2.33.4" 2024-04-22T22:24:11.420 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724566 got 167503724570 for osd.5 2024-04-22T22:24:11.420 DEBUG:teuthology.parallel:result is None 2024-04-22T22:24:11.506 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:24:11 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:24:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:24:11.662 INFO:teuthology.orchestra.run.smithi067.stderr:Inferring config /var/lib/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T22:24:11.748 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:11 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/2694777131' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-22T22:24:11.748 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:11 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/2694777131' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-22T22:24:11.805 INFO:teuthology.orchestra.run.smithi067.stdout:77309411368 2024-04-22T22:24:11.948 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:11 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/2694777131' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-22T22:24:12.459 INFO:teuthology.orchestra.run.smithi067.stdout:34359738418 2024-04-22T22:24:12.758 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:12 smithi067 ceph-mon[21875]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:12.758 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:12 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/562318422' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-22T22:24:12.758 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:12 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/481368410' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-22T22:24:12.758 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:12 smithi067 ceph-mon[26633]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:12.758 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:12 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/562318422' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-22T22:24:12.758 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:12 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/481368410' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-22T22:24:13.006 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:12 smithi086 ceph-mon[22689]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:13.006 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:12 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/562318422' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-22T22:24:13.006 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:12 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/481368410' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-22T22:24:13.878 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:13 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:24:13.878 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:13 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:24:13.878 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:13 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:24:13.878 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:13 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:24:14.006 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:13 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:24:14.006 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:13 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:24:14.078 INFO:teuthology.orchestra.run.smithi067.stdout:111669149732 2024-04-22T22:24:14.716 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149728 got 111669149732 for osd.3 2024-04-22T22:24:14.716 DEBUG:teuthology.parallel:result is None 2024-04-22T22:24:15.006 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:14 smithi086 ceph-mon[22689]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:15.006 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:14 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/2637219221' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-22T22:24:15.007 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:14 smithi067 ceph-mon[26633]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:15.007 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:14 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/2637219221' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-22T22:24:15.007 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:14 smithi067 ceph-mon[21875]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:15.007 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:14 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/2637219221' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-22T22:24:15.483 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411364 got 77309411368 for osd.2 2024-04-22T22:24:15.483 DEBUG:teuthology.parallel:result is None 2024-04-22T22:24:16.859 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:16 smithi067 ceph-mon[21875]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:16.859 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:16 smithi067 ceph-mon[26633]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:16.907 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738414 got 34359738418 for osd.0 2024-04-22T22:24:16.908 DEBUG:teuthology.parallel:result is None 2024-04-22T22:24:17.006 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:16 smithi086 ceph-mon[22689]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:17.274 INFO:teuthology.orchestra.run.smithi067.stdout:236223201298 2024-04-22T22:24:18.000 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:17 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/864401788' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-22T22:24:18.000 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:17 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/864401788' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-22T22:24:18.006 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:17 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/864401788' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-22T22:24:18.026 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201294 got 236223201298 for osd.7 2024-04-22T22:24:18.026 DEBUG:teuthology.parallel:result is None 2024-04-22T22:24:18.657 INFO:teuthology.orchestra.run.smithi067.stdout:55834574894 2024-04-22T22:24:18.658 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:18 smithi067 ceph-mon[21875]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:18.658 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:18 smithi067 ceph-mon[26633]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:19.006 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:18 smithi086 ceph-mon[22689]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:19.343 INFO:teuthology.orchestra.run.smithi067.stdout:133143986207 2024-04-22T22:24:19.931 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:19 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/3507183316' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-22T22:24:19.932 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:19 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/2969132235' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-22T22:24:19.932 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:19 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/3507183316' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-22T22:24:19.932 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:19 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/2969132235' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-22T22:24:20.006 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:19 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/3507183316' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-22T22:24:20.006 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:19 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/2969132235' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-22T22:24:20.257 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:20 smithi067 systemd[1]: Starting Ceph alertmanager.a for 1ab1a3a8-00f6-11ef-bc93-c7b262605968... 2024-04-22T22:24:20.724 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574890 got 55834574894 for osd.1 2024-04-22T22:24:20.725 DEBUG:teuthology.parallel:result is None 2024-04-22T22:24:21.006 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:20 smithi086 ceph-mon[22689]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:21.007 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:20 smithi067 ceph-mon[21875]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:21.007 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:20 smithi067 ceph-mon[26633]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:21.008 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:24:20 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:24:20] "GET /metrics HTTP/1.1" 200 191487 "" "Prometheus/2.33.4" 2024-04-22T22:24:21.158 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986203 got 133143986207 for osd.4 2024-04-22T22:24:21.159 DEBUG:teuthology.parallel:result is None 2024-04-22T22:24:21.159 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-22T22:24:21.159 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-22T22:24:21.594 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:21 smithi067 podman[44754]: 2024-04-22 22:24:21.217454845 +0000 UTC m=+0.019823225 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-04-22T22:24:21.594 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:21 smithi067 podman[44754]: 2024-04-22 22:24:21.33630715 +0000 UTC m=+0.138675529 volume create f42f03ef11640a2ad9f062167873fb06797c361f7648a958cc90755ee7bb32d0 2024-04-22T22:24:21.594 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:21 smithi067 podman[44754]: 2024-04-22 22:24:21.45364459 +0000 UTC m=+0.256012969 container create cabfcc88c704345c6aa854bd1eb602c1fd87e04e2d98679d8d9c68fdbb56b711 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T22:24:21.756 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:24:21 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:24:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:24:21.952 INFO:teuthology.orchestra.run.smithi067.stderr:Inferring config /var/lib/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T22:24:22.085 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:21 smithi067 podman[44754]: 2024-04-22 22:24:21.773853173 +0000 UTC m=+0.576221552 container init cabfcc88c704345c6aa854bd1eb602c1fd87e04e2d98679d8d9c68fdbb56b711 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T22:24:22.085 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:21 smithi067 podman[44754]: 2024-04-22 22:24:21.776926959 +0000 UTC m=+0.579295338 container start cabfcc88c704345c6aa854bd1eb602c1fd87e04e2d98679d8d9c68fdbb56b711 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T22:24:22.085 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:21 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager-a[44785]: level=info ts=2024-04-22T22:24:21.787Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-04-22T22:24:22.085 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:21 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager-a[44785]: level=info ts=2024-04-22T22:24:21.787Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-04-22T22:24:22.085 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:21 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager-a[44785]: level=info ts=2024-04-22T22:24:21.789Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.67 port=9094 2024-04-22T22:24:22.085 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:21 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager-a[44785]: level=info ts=2024-04-22T22:24:21.790Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-22T22:24:22.085 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:21 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager-a[44785]: level=info ts=2024-04-22T22:24:21.816Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-22T22:24:22.085 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:21 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager-a[44785]: level=info ts=2024-04-22T22:24:21.816Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-22T22:24:22.085 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:21 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager-a[44785]: level=info ts=2024-04-22T22:24:21.818Z caller=main.go:518 msg=Listening address=:9093 2024-04-22T22:24:22.085 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:21 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager-a[44785]: level=info ts=2024-04-22T22:24:21.818Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-04-22T22:24:22.085 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:21 smithi067 bash[44754]: cabfcc88c704345c6aa854bd1eb602c1fd87e04e2d98679d8d9c68fdbb56b711 2024-04-22T22:24:22.085 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:21 smithi067 systemd[1]: Started Ceph alertmanager.a for 1ab1a3a8-00f6-11ef-bc93-c7b262605968. 2024-04-22T22:24:23.229 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:22 smithi067 ceph-mon[21875]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:23.229 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:22 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:23.229 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:22 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:23.230 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:22 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:23.230 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:22 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T22:24:23.230 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:22 smithi067 ceph-mon[21875]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T22:24:23.230 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:22 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:23.230 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:22 smithi067 ceph-mon[21875]: Deploying daemon grafana.a on smithi086 2024-04-22T22:24:23.230 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:22 smithi067 ceph-mon[26633]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:23.230 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:22 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:23.230 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:22 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:23.230 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:22 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:23.230 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:22 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T22:24:23.230 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:22 smithi067 ceph-mon[26633]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T22:24:23.230 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:22 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:23.230 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:22 smithi067 ceph-mon[26633]: Deploying daemon grafana.a on smithi086 2024-04-22T22:24:23.257 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:22 smithi086 ceph-mon[22689]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:23.257 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:22 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:23.257 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:22 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:23.257 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:22 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:23.257 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:22 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T22:24:23.257 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:22 smithi086 ceph-mon[22689]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-22T22:24:23.257 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:22 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:23.257 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:22 smithi086 ceph-mon[22689]: Deploying daemon grafana.a on smithi086 2024-04-22T22:24:23.619 INFO:teuthology.orchestra.run.smithi067.stdout: 2024-04-22T22:24:23.620 INFO:teuthology.orchestra.run.smithi067.stderr:dumped all 2024-04-22T22:24:24.232 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:23 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager-a[44785]: level=info ts=2024-04-22T22:24:23.791Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000876104s 2024-04-22T22:24:24.294 INFO:teuthology.orchestra.run.smithi067.stdout:{"pg_ready":true,"pg_map":{"version":38,"stamp":"2024-04-22T22:24:23.344834+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48968,"kb_used_data":6152,"kb_used_omap":0,"kb_used_meta":42752,"kb_avail":749912248,"statfs":{"total":767960285184,"available":767910141952,"internally_reserved":0,"allocated":6299648,"data_stored":3510880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43778048},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002073"},"pg_stats":[{"pgid":"1.0","version":"60'128","reported_seq":267,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-04-22T22:23:13.413669+0000","last_change":"2024-04-22T22:22:57.571050+0000","last_active":"2024-04-22T22:23:13.413669+0000","last_peered":"2024-04-22T22:23:13.413669+0000","last_clean":"2024-04-22T22:23:13.413669+0000","last_became_active":"2024-04-22T22:22:57.570524+0000","last_became_peered":"2024-04-22T22:22:57.570524+0000","last_unstale":"2024-04-22T22:23:13.413669+0000","last_undegraded":"2024-04-22T22:23:13.413669+0000","last_fullsized":"2024-04-22T22:23:13.413669+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-22T22:21:02.937562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-22T22:21:02.937562+0000","last_clean_scrub_stamp":"2024-04-22T22:21:02.937562+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-24T00:08:36.299142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201299,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6092,"kb_used_data":1092,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739060,"statfs":{"total":95995035648,"available":95988797440,"internally_reserved":0,"allocated":1118208,"data_stored":769315,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 22:23:54 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.58599999999999997}]},{"osd":1,"last update":"Mon Apr 22 22:23:54 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.60599999999999998}]},{"osd":2,"last update":"Mon Apr 22 22:23:54 2024","interfaces":[{"interface":"back","average":{"1min":1.127,"5min":1.127,"15min":1.127},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":10.815,"5min":10.815,"15min":10.815},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.70799999999999996}]},{"osd":3,"last update":"Mon Apr 22 22:23:54 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.627}]},{"osd":4,"last update":"Mon Apr 22 22:23:54 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.63900000000000001}]},{"osd":5,"last update":"Mon Apr 22 22:23:54 2024","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.55300000000000005}]},{"osd":6,"last update":"Mon Apr 22 22:23:54 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.69499999999999995}]}]},{"osd":6,"up_from":47,"seq":201863462936,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6152,"kb_used_data":1088,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739000,"statfs":{"total":95995035648,"available":95988736000,"internally_reserved":0,"allocated":1114112,"data_stored":769000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 22:23:31 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.78700000000000003},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.73399999999999999}]},{"osd":1,"last update":"Mon Apr 22 22:23:31 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.745},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.64000000000000001}]},{"osd":2,"last update":"Mon Apr 22 22:23:31 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.80300000000000005},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.622}]},{"osd":3,"last update":"Mon Apr 22 22:23:31 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.76600000000000001}]},{"osd":4,"last update":"Mon Apr 22 22:23:31 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.44},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.51300000000000001}]},{"osd":5,"last update":"Mon Apr 22 22:23:31 2024","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.45400000000000001}]},{"osd":7,"last update":"Mon Apr 22 22:23:54 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.55200000000000005}]}]},{"osd":1,"up_from":13,"seq":55834574895,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6344,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738808,"statfs":{"total":95995035648,"available":95988539392,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 22:23:41 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.377,"15min":0.36099999999999999},"min":{"1min":0.27000000000000002,"5min":0.223,"15min":0.223},"max":{"1min":0.68300000000000005,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.372},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.39500000000000002,"15min":0.375},"min":{"1min":0.27000000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.34100000000000003}]},{"osd":2,"last update":"Mon Apr 22 22:24:06 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.40699999999999997,"15min":0.39500000000000002},"min":{"1min":0.27000000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.44600000000000001,"15min":0.43099999999999999},"min":{"1min":0.29799999999999999,"5min":0.248,"15min":0.248},"max":{"1min":0.98399999999999999,"5min":0.98399999999999999,"15min":0.98399999999999999},"last":0.41899999999999998}]},{"osd":3,"last update":"Mon Apr 22 22:23:30 2024","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.503,"15min":0.50700000000000001},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.40200000000000002}]},{"osd":4,"last update":"Mon Apr 22 22:23:46 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.58099999999999996,"15min":0.57899999999999996},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.60099999999999998,"15min":0.60299999999999998},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.58999999999999997}]},{"osd":5,"last update":"Mon Apr 22 22:24:11 2024","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.622,"15min":0.622},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.65600000000000003,"15min":0.66200000000000003},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.96899999999999997,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.58199999999999996}]},{"osd":6,"last update":"Mon Apr 22 22:23:33 2024","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.878,"5min":0.878,"15min":0.878},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.65100000000000002}]},{"osd":7,"last update":"Mon Apr 22 22:23:58 2024","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.96299999999999997,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.69899999999999995}]}]},{"osd":0,"up_from":8,"seq":34359738420,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6928,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738224,"statfs":{"total":95995035648,"available":95987941376,"internally_reserved":0,"allocated":1122304,"data_stored":769588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Apr 22 22:23:41 2024","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.36799999999999999,"15min":0.35799999999999998},"min":{"1min":0.26500000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.60599999999999998,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.28499999999999998},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.377,"15min":0.35399999999999998},"min":{"1min":0.23200000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.60699999999999998,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.38}]},{"osd":2,"last update":"Mon Apr 22 22:24:05 2024","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.41999999999999998,"15min":0.42199999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.54500000000000004,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.41299999999999998,"15min":0.40799999999999997},"min":{"1min":0.30099999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.33000000000000002}]},{"osd":3,"last update":"Mon Apr 22 22:23:25 2024","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.48099999999999998,"15min":0.47999999999999998},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.45700000000000002,"15min":0.45900000000000002},"min":{"1min":0.317,"5min":0.315,"15min":0.315},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.43099999999999999}]},{"osd":4,"last update":"Mon Apr 22 22:23:48 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.58899999999999997,"15min":0.59299999999999997},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.022,"5min":1.022,"15min":1.022},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.67400000000000004}]},{"osd":5,"last update":"Mon Apr 22 22:24:14 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.62,"15min":0.626},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.60799999999999998,"15min":0.61499999999999999},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.73899999999999999,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.55200000000000005}]},{"osd":6,"last update":"Mon Apr 22 22:23:32 2024","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.98699999999999999,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.63200000000000001}]},{"osd":7,"last update":"Mon Apr 22 22:23:57 2024","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.60699999999999998}]}]},{"osd":2,"up_from":18,"seq":77309411370,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6280,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738872,"statfs":{"total":95995035648,"available":95988604928,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 22:24:07 2024","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.38100000000000001,"15min":0.36799999999999999},"min":{"1min":0.26900000000000002,"5min":0.186,"15min":0.186},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.42699999999999999,"5min":0.40000000000000002,"15min":0.38300000000000001},"min":{"1min":0.307,"5min":0.247,"15min":0.247},"max":{"1min":0.57599999999999996,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.497}]},{"osd":1,"last update":"Mon Apr 22 22:24:07 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.39100000000000001,"15min":0.38300000000000001},"min":{"1min":0.27300000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.53700000000000003,"5min":0.621,"15min":0.621},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.439,"5min":0.44,"15min":0.42799999999999999},"min":{"1min":0.32500000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.61199999999999999,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.57799999999999996}]},{"osd":3,"last update":"Mon Apr 22 22:23:28 2024","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.51100000000000001,"15min":0.51700000000000002},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.433,"5min":0.46899999999999997,"15min":0.47499999999999998},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.56799999999999995}]},{"osd":4,"last update":"Mon Apr 22 22:23:53 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.58199999999999996,"15min":0.58999999999999997},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.76300000000000001,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.77000000000000002},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.63500000000000001,"15min":0.64200000000000002},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.751,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.79300000000000004}]},{"osd":5,"last update":"Mon Apr 22 22:24:14 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.625,"15min":0.625},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.83199999999999996,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.83199999999999996},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.41099999999999998,"5min":0.372,"15min":0.372},"max":{"1min":0.84099999999999997,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.84099999999999997}]},{"osd":6,"last update":"Mon Apr 22 22:23:33 2024","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.747},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.78400000000000003}]},{"osd":7,"last update":"Mon Apr 22 22:23:57 2024","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.82699999999999996},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.88200000000000001}]}]},{"osd":3,"up_from":26,"seq":111669149734,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5704,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739448,"statfs":{"total":95995035648,"available":95989194752,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 22:23:25 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.371,"15min":0.36899999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"last":0.37},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.39900000000000002,"15min":0.40100000000000002},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.52400000000000002,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.27600000000000002}]},{"osd":1,"last update":"Mon Apr 22 22:23:25 2024","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.48899999999999999,"15min":0.49299999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.65800000000000003,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.44700000000000001,"15min":0.45500000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.48899999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.33200000000000002}]},{"osd":2,"last update":"Mon Apr 22 22:23:25 2024","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.45400000000000001,"15min":0.45400000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.71199999999999997,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.66900000000000004,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.40200000000000002}]},{"osd":4,"last update":"Mon Apr 22 22:23:47 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.56699999999999995,"15min":0.57099999999999995},"min":{"1min":0.434,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.56599999999999995,"15min":0.56899999999999995},"min":{"1min":0.43099999999999999,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.48299999999999998}]},{"osd":5,"last update":"Mon Apr 22 22:24:12 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.56200000000000006,"15min":0.55400000000000005},"min":{"1min":0.51900000000000002,"5min":0.376,"15min":0.376},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.58499999999999996,"15min":0.58299999999999996},"min":{"1min":0.438,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.495}]},{"osd":6,"last update":"Mon Apr 22 22:23:35 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.63200000000000001}]},{"osd":7,"last update":"Mon Apr 22 22:23:57 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.58199999999999996}]}]},{"osd":4,"up_from":31,"seq":133143986208,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5768,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739384,"statfs":{"total":95995035648,"available":95989129216,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 22:23:45 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.54800000000000004,"15min":0.55300000000000005},"min":{"1min":0.35599999999999998,"5min":0.307,"15min":0.307},"max":{"1min":0.68400000000000005,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.503,"5min":0.54200000000000004,"15min":0.54800000000000004},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.66000000000000003,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.624}]},{"osd":1,"last update":"Mon Apr 22 22:23:45 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.53400000000000003,"15min":0.53900000000000003},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.61599999999999999,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.54200000000000004,"15min":0.54600000000000004},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.66700000000000004,"5min":0.88,"15min":0.88},"last":0.54400000000000004}]},{"osd":2,"last update":"Mon Apr 22 22:23:45 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.56200000000000006,"15min":0.56999999999999995},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.65900000000000003,"5min":0.876,"15min":0.876},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.56100000000000005,"15min":0.56599999999999995},"min":{"1min":0.42799999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.67600000000000005,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.59299999999999997}]},{"osd":3,"last update":"Mon Apr 22 22:23:45 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.53200000000000003,"15min":0.53400000000000003},"min":{"1min":0.36899999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.69499999999999995,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.56999999999999995,"15min":0.57899999999999996},"min":{"1min":0.39900000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.64000000000000001,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.57199999999999995}]},{"osd":5,"last update":"Mon Apr 22 22:24:12 2024","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.39000000000000001,"15min":0.378},"min":{"1min":0.28399999999999997,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.40899999999999997,"15min":0.40000000000000002},"min":{"1min":0.29499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.72399999999999998,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.40999999999999998}]},{"osd":6,"last update":"Mon Apr 22 22:23:34 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.443,"15min":0.443},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.55700000000000005}]},{"osd":7,"last update":"Mon Apr 22 22:23:57 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.50600000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724572,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5700,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739452,"statfs":{"total":95995035648,"available":95989198848,"internally_reserved":0,"allocated":585728,"data_stored":240337,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 22:24:10 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.75700000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.53300000000000003,"15min":0.52900000000000003},"min":{"1min":0.41999999999999998,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.53800000000000003}]},{"osd":1,"last update":"Mon Apr 22 22:24:10 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.54800000000000004,"15min":0.55100000000000005},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.73799999999999999,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.499},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.51100000000000001,"15min":0.503},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.49199999999999999}]},{"osd":2,"last update":"Mon Apr 22 22:24:10 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.53200000000000003,"15min":0.53000000000000003},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.73299999999999998,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.63},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.55300000000000005,"15min":0.55000000000000004},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.58299999999999996}]},{"osd":3,"last update":"Mon Apr 22 22:24:10 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.52500000000000002,"15min":0.52300000000000002},"min":{"1min":0.39700000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.56100000000000005,"15min":0.55500000000000005},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.78500000000000003,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.64600000000000002}]},{"osd":4,"last update":"Mon Apr 22 22:24:10 2024","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.38700000000000001,"15min":0.373},"min":{"1min":0.32700000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.505,"5min":0.40100000000000002,"15min":0.38400000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.56699999999999995}]},{"osd":6,"last update":"Mon Apr 22 22:23:31 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.66900000000000004}]},{"osd":7,"last update":"Mon Apr 22 22:23:58 2024","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.69399999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-22T22:24:24.296 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-22T22:24:24.508 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:24 smithi067 ceph-mon[21875]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:24.508 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:24 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:24.508 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:24 smithi067 ceph-mon[21875]: from='client.14622 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T22:24:24.508 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:24 smithi067 ceph-mon[26633]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:24.508 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:24 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:24.508 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:24 smithi067 ceph-mon[26633]: from='client.14622 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T22:24:24.521 INFO:teuthology.orchestra.run.smithi067.stderr:Inferring config /var/lib/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T22:24:24.756 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:24 smithi086 ceph-mon[22689]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:24.756 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:24 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:24.756 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:24 smithi086 ceph-mon[22689]: from='client.14622 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T22:24:26.239 INFO:teuthology.orchestra.run.smithi067.stdout: 2024-04-22T22:24:26.240 INFO:teuthology.orchestra.run.smithi067.stderr:dumped all 2024-04-22T22:24:26.948 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:26 smithi067 ceph-mon[21875]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:26.948 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:26 smithi067 ceph-mon[21875]: from='client.24520 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T22:24:26.948 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:26 smithi067 ceph-mon[26633]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:26.948 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:26 smithi067 ceph-mon[26633]: from='client.24520 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T22:24:26.976 INFO:teuthology.orchestra.run.smithi067.stdout:{"pg_ready":true,"pg_map":{"version":39,"stamp":"2024-04-22T22:24:25.345150+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48968,"kb_used_data":6152,"kb_used_omap":0,"kb_used_meta":42752,"kb_avail":749912248,"statfs":{"total":767960285184,"available":767910141952,"internally_reserved":0,"allocated":6299648,"data_stored":3510880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43778048},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002014"},"pg_stats":[{"pgid":"1.0","version":"60'128","reported_seq":267,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-04-22T22:23:13.413669+0000","last_change":"2024-04-22T22:22:57.571050+0000","last_active":"2024-04-22T22:23:13.413669+0000","last_peered":"2024-04-22T22:23:13.413669+0000","last_clean":"2024-04-22T22:23:13.413669+0000","last_became_active":"2024-04-22T22:22:57.570524+0000","last_became_peered":"2024-04-22T22:22:57.570524+0000","last_unstale":"2024-04-22T22:23:13.413669+0000","last_undegraded":"2024-04-22T22:23:13.413669+0000","last_fullsized":"2024-04-22T22:23:13.413669+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-22T22:21:02.937562+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-22T22:21:02.937562+0000","last_clean_scrub_stamp":"2024-04-22T22:21:02.937562+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-24T00:08:36.299142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201300,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6092,"kb_used_data":1092,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739060,"statfs":{"total":95995035648,"available":95988797440,"internally_reserved":0,"allocated":1118208,"data_stored":769315,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 22:23:54 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.436},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.42699999999999999}]},{"osd":1,"last update":"Mon Apr 22 22:23:54 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.40999999999999998}]},{"osd":2,"last update":"Mon Apr 22 22:23:54 2024","interfaces":[{"interface":"back","average":{"1min":1.127,"5min":1.127,"15min":1.127},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":10.815,"5min":10.815,"15min":10.815},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.44600000000000001}]},{"osd":3,"last update":"Mon Apr 22 22:23:54 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.46300000000000002}]},{"osd":4,"last update":"Mon Apr 22 22:23:54 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"last":0.311},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.371}]},{"osd":5,"last update":"Mon Apr 22 22:23:54 2024","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.38}]},{"osd":6,"last update":"Mon Apr 22 22:23:54 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.41999999999999998}]}]},{"osd":6,"up_from":47,"seq":201863462937,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6152,"kb_used_data":1088,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739000,"statfs":{"total":95995035648,"available":95988736000,"internally_reserved":0,"allocated":1114112,"data_stored":769000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 22:23:31 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.378}]},{"osd":1,"last update":"Mon Apr 22 22:23:31 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.40500000000000003}]},{"osd":2,"last update":"Mon Apr 22 22:23:31 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.434},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.41999999999999998}]},{"osd":3,"last update":"Mon Apr 22 22:23:31 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.34399999999999997}]},{"osd":4,"last update":"Mon Apr 22 22:23:31 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.28799999999999998}]},{"osd":5,"last update":"Mon Apr 22 22:23:31 2024","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.443},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.32400000000000001}]},{"osd":7,"last update":"Mon Apr 22 22:23:54 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.35599999999999998}]}]},{"osd":1,"up_from":13,"seq":55834574896,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6344,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738808,"statfs":{"total":95995035648,"available":95988539392,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 22:23:41 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.377,"15min":0.36099999999999999},"min":{"1min":0.27000000000000002,"5min":0.223,"15min":0.223},"max":{"1min":0.68300000000000005,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.39500000000000002,"15min":0.375},"min":{"1min":0.27000000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.36199999999999999}]},{"osd":2,"last update":"Mon Apr 22 22:24:06 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.40699999999999997,"15min":0.39500000000000002},"min":{"1min":0.27000000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.44600000000000001,"15min":0.43099999999999999},"min":{"1min":0.29799999999999999,"5min":0.248,"15min":0.248},"max":{"1min":0.98399999999999999,"5min":0.98399999999999999,"15min":0.98399999999999999},"last":0.33800000000000002}]},{"osd":3,"last update":"Mon Apr 22 22:23:30 2024","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.503,"15min":0.50700000000000001},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.34300000000000003}]},{"osd":4,"last update":"Mon Apr 22 22:23:46 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.58099999999999996,"15min":0.57899999999999996},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.60099999999999998,"15min":0.60299999999999998},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.44}]},{"osd":5,"last update":"Mon Apr 22 22:24:11 2024","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.622,"15min":0.622},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.65600000000000003,"15min":0.66200000000000003},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.96899999999999997,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.44900000000000001}]},{"osd":6,"last update":"Mon Apr 22 22:23:33 2024","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.878,"5min":0.878,"15min":0.878},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.47799999999999998}]},{"osd":7,"last update":"Mon Apr 22 22:23:58 2024","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.96299999999999997,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.51600000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738421,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6928,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738224,"statfs":{"total":95995035648,"available":95987941376,"internally_reserved":0,"allocated":1122304,"data_stored":769588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Apr 22 22:23:41 2024","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.36799999999999999,"15min":0.35799999999999998},"min":{"1min":0.26500000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.60599999999999998,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.442},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.377,"15min":0.35399999999999998},"min":{"1min":0.23200000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.60699999999999998,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.54800000000000004}]},{"osd":2,"last update":"Mon Apr 22 22:24:05 2024","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.41999999999999998,"15min":0.42199999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.54500000000000004,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.41299999999999998,"15min":0.40799999999999997},"min":{"1min":0.30099999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.46800000000000003}]},{"osd":3,"last update":"Mon Apr 22 22:23:25 2024","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.48099999999999998,"15min":0.47999999999999998},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.45700000000000002,"15min":0.45900000000000002},"min":{"1min":0.317,"5min":0.315,"15min":0.315},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.51400000000000001}]},{"osd":4,"last update":"Mon Apr 22 22:23:48 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.58899999999999997,"15min":0.59299999999999997},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.022,"5min":1.022,"15min":1.022},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.61399999999999999}]},{"osd":5,"last update":"Mon Apr 22 22:24:14 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.62,"15min":0.626},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"last":0.73099999999999998},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.60799999999999998,"15min":0.61499999999999999},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.73899999999999999,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.69599999999999995}]},{"osd":6,"last update":"Mon Apr 22 22:23:32 2024","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.98699999999999999,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.70299999999999996}]},{"osd":7,"last update":"Mon Apr 22 22:23:57 2024","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.73599999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411371,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6280,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738872,"statfs":{"total":95995035648,"available":95988604928,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 22:24:07 2024","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.38100000000000001,"15min":0.36799999999999999},"min":{"1min":0.26900000000000002,"5min":0.186,"15min":0.186},"max":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.42699999999999999,"5min":0.40000000000000002,"15min":0.38300000000000001},"min":{"1min":0.307,"5min":0.247,"15min":0.247},"max":{"1min":0.57599999999999996,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.50800000000000001}]},{"osd":1,"last update":"Mon Apr 22 22:24:07 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.39100000000000001,"15min":0.38300000000000001},"min":{"1min":0.27300000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.53700000000000003,"5min":0.621,"15min":0.621},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.439,"5min":0.44,"15min":0.42799999999999999},"min":{"1min":0.32500000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.61199999999999999,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.45100000000000001}]},{"osd":3,"last update":"Mon Apr 22 22:23:28 2024","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.51100000000000001,"15min":0.51700000000000002},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.433,"5min":0.46899999999999997,"15min":0.47499999999999998},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.435}]},{"osd":4,"last update":"Mon Apr 22 22:23:53 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.58199999999999996,"15min":0.58999999999999997},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.76300000000000001,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.63500000000000001,"15min":0.64200000000000002},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.751,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.61899999999999999}]},{"osd":5,"last update":"Mon Apr 22 22:24:14 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.625,"15min":0.625},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.83199999999999996,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.41099999999999998,"5min":0.372,"15min":0.372},"max":{"1min":0.84099999999999997,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.67400000000000004}]},{"osd":6,"last update":"Mon Apr 22 22:23:33 2024","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.70299999999999996}]},{"osd":7,"last update":"Mon Apr 22 22:23:57 2024","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.73799999999999999},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.61299999999999999}]}]},{"osd":3,"up_from":26,"seq":111669149735,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5704,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739448,"statfs":{"total":95995035648,"available":95989194752,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 22:23:25 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.371,"15min":0.36899999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"last":0.37},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.39900000000000002,"15min":0.40100000000000002},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.52400000000000002,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.27600000000000002}]},{"osd":1,"last update":"Mon Apr 22 22:23:25 2024","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.48899999999999999,"15min":0.49299999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.65800000000000003,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.44700000000000001,"15min":0.45500000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.48899999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.33200000000000002}]},{"osd":2,"last update":"Mon Apr 22 22:23:25 2024","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.45400000000000001,"15min":0.45400000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.71199999999999997,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.66900000000000004,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.40200000000000002}]},{"osd":4,"last update":"Mon Apr 22 22:23:47 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.56699999999999995,"15min":0.57099999999999995},"min":{"1min":0.434,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.56599999999999995,"15min":0.56899999999999995},"min":{"1min":0.43099999999999999,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.48299999999999998}]},{"osd":5,"last update":"Mon Apr 22 22:24:12 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.56200000000000006,"15min":0.55400000000000005},"min":{"1min":0.51900000000000002,"5min":0.376,"15min":0.376},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.58499999999999996,"15min":0.58299999999999996},"min":{"1min":0.438,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.495}]},{"osd":6,"last update":"Mon Apr 22 22:23:35 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.63200000000000001}]},{"osd":7,"last update":"Mon Apr 22 22:23:57 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.58199999999999996}]}]},{"osd":4,"up_from":31,"seq":133143986209,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5768,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739384,"statfs":{"total":95995035648,"available":95989129216,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 22:23:45 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.54800000000000004,"15min":0.55300000000000005},"min":{"1min":0.35599999999999998,"5min":0.307,"15min":0.307},"max":{"1min":0.68400000000000005,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.502},{"interface":"front","average":{"1min":0.503,"5min":0.54200000000000004,"15min":0.54800000000000004},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.66000000000000003,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.432}]},{"osd":1,"last update":"Mon Apr 22 22:23:45 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.53400000000000003,"15min":0.53900000000000003},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.61599999999999999,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.54200000000000004,"15min":0.54600000000000004},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.66700000000000004,"5min":0.88,"15min":0.88},"last":0.34000000000000002}]},{"osd":2,"last update":"Mon Apr 22 22:23:45 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.56200000000000006,"15min":0.56999999999999995},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.65900000000000003,"5min":0.876,"15min":0.876},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.56100000000000005,"15min":0.56599999999999995},"min":{"1min":0.42799999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.67600000000000005,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.496}]},{"osd":3,"last update":"Mon Apr 22 22:23:45 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.53200000000000003,"15min":0.53400000000000003},"min":{"1min":0.36899999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.69499999999999995,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.371},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.56999999999999995,"15min":0.57899999999999996},"min":{"1min":0.39900000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.64000000000000001,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.35999999999999999}]},{"osd":5,"last update":"Mon Apr 22 22:24:12 2024","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.39000000000000001,"15min":0.378},"min":{"1min":0.28399999999999997,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.40899999999999997,"15min":0.40000000000000002},"min":{"1min":0.29499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.72399999999999998,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.32500000000000001}]},{"osd":6,"last update":"Mon Apr 22 22:23:34 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.443,"15min":0.443},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.51500000000000001}]},{"osd":7,"last update":"Mon Apr 22 22:23:57 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.53000000000000003}]}]},{"osd":5,"up_from":39,"seq":167503724573,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5700,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739452,"statfs":{"total":95995035648,"available":95989198848,"internally_reserved":0,"allocated":585728,"data_stored":240337,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Apr 22 22:24:10 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.75700000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.53300000000000003,"15min":0.52900000000000003},"min":{"1min":0.41999999999999998,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.437}]},{"osd":1,"last update":"Mon Apr 22 22:24:10 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.54800000000000004,"15min":0.55100000000000005},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.73799999999999999,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.51100000000000001,"15min":0.503},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.373}]},{"osd":2,"last update":"Mon Apr 22 22:24:10 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.53200000000000003,"15min":0.53000000000000003},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.73299999999999998,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.55300000000000005,"15min":0.55000000000000004},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.34899999999999998}]},{"osd":3,"last update":"Mon Apr 22 22:24:10 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.52500000000000002,"15min":0.52300000000000002},"min":{"1min":0.39700000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.56100000000000005,"15min":0.55500000000000005},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.78500000000000003,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.38400000000000001}]},{"osd":4,"last update":"Mon Apr 22 22:24:10 2024","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.38700000000000001,"15min":0.373},"min":{"1min":0.32700000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.505,"5min":0.40100000000000002,"15min":0.38400000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.28499999999999998}]},{"osd":6,"last update":"Mon Apr 22 22:23:31 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.45900000000000002}]},{"osd":7,"last update":"Mon Apr 22 22:23:58 2024","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.44400000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-22T22:24:26.977 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-22T22:24:26.978 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-22T22:24:26.978 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-22T22:24:26.978 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-22T22:24:27.006 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:26 smithi086 ceph-mon[22689]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:27.006 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:26 smithi086 ceph-mon[22689]: from='client.24520 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-22T22:24:27.202 INFO:teuthology.orchestra.run.smithi067.stderr:Inferring config /var/lib/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/mon.a/config 2024-04-22T22:24:28.757 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:28 smithi067 ceph-mon[21875]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:28.758 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:28 smithi067 ceph-mon[26633]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:29.006 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:28 smithi086 ceph-mon[22689]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:29.248 INFO:teuthology.orchestra.run.smithi067.stdout: 2024-04-22T22:24:29.249 INFO:teuthology.orchestra.run.smithi067.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-22T22:24:29.853 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:29 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/2935424856' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-22T22:24:29.854 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:29 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/2935424856' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-22T22:24:29.879 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-22T22:24:29.879 INFO:tasks.cephadm:Setup complete, yielding 2024-04-22T22:24:29.879 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-22T22:24:29.889 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi067.front.sepia.ceph.com 2024-04-22T22:24:29.889 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -- bash -c 'ceph config set mgr mgr/cephadm/use_repo_digest false --force' 2024-04-22T22:24:30.006 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:29 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/2935424856' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-22T22:24:31.006 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:30 smithi086 ceph-mon[22689]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:31.008 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:30 smithi067 ceph-mon[21875]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:31.008 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:30 smithi067 ceph-mon[26633]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:31.008 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:24:30 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:24:30] "GET /metrics HTTP/1.1" 200 191483 "" "Prometheus/2.33.4" 2024-04-22T22:24:31.710 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:24:31 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:24:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:24:32.229 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:31 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager-a[44785]: level=info ts=2024-04-22T22:24:31.792Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.001805583s 2024-04-22T22:24:32.361 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-22T22:24:32.373 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi067.front.sepia.ceph.com 2024-04-22T22:24:32.373 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -e sha1=430e09df97c8fc7dc2b2ae424f68ed11366c540f -- bash -c 'radosgw-admin realm create --rgw-realm=r --default' 2024-04-22T22:24:33.007 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:32 smithi086 ceph-mon[22689]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:33.007 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:32 smithi086 ceph-mon[22689]: from='client.? ' entity='client.admin' 2024-04-22T22:24:33.008 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:32 smithi067 ceph-mon[21875]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:33.008 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:32 smithi067 ceph-mon[21875]: from='client.? ' entity='client.admin' 2024-04-22T22:24:33.008 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:32 smithi067 ceph-mon[26633]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:33.008 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:32 smithi067 ceph-mon[26633]: from='client.? ' entity='client.admin' 2024-04-22T22:24:34.007 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:33 smithi086 podman[38693]: 2024-04-22 22:24:33.982699094 +0000 UTC m=+0.121117090 container create 5d6c83f8480c9a452d14aab2194c813ff3d4be802405ce2ea435a5d459b911b3 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a, vcs-type=git, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, maintainer=Paul Cuzner , build-date=2022-03-28T10:36:18.413762, distribution-scope=public, description=Ceph Grafana Container, io.openshift.expose-services=, io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=236.1648460182, name=ubi8, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.24.2, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container) 2024-04-22T22:24:34.324 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 podman[38693]: 2024-04-22 22:24:34.193971006 +0000 UTC m=+0.332389013 container init 5d6c83f8480c9a452d14aab2194c813ff3d4be802405ce2ea435a5d459b911b3 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, architecture=x86_64, build-date=2022-03-28T10:36:18.413762, distribution-scope=public, vcs-type=git, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, vendor=Red Hat, Inc., release=236.1648460182, version=8.5, io.openshift.expose-services=, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.buildah.version=1.24.2) 2024-04-22T22:24:34.324 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 podman[38693]: 2024-04-22 22:24:34.196772983 +0000 UTC m=+0.335190982 container start 5d6c83f8480c9a452d14aab2194c813ff3d4be802405ce2ea435a5d459b911b3 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a, version=8.5, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, release=236.1648460182, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=ubi8-container, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.buildah.version=1.24.2, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, build-date=2022-03-28T10:36:18.413762, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-04-22T22:24:34.325 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2024-04-22T22:24:34.325 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2024-04-22T22:24:34.325 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2024-04-22T22:24:34.325 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2024-04-22T22:24:34.325 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2024-04-22T22:24:34.325 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2024-04-22T22:24:34.325 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-04-22T22:24:34.326 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-04-22T22:24:34.326 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2024-04-22T22:24:34.326 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2024-04-22T22:24:34.326 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2024-04-22T22:24:34.326 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2024-04-22T22:24:34.326 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2024-04-22T22:24:34.326 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="App mode production" logger=settings 2024-04-22T22:24:34.326 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2024-04-22T22:24:34.326 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-04-22T22:24:34.326 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Starting DB migrations" logger=migrator 2024-04-22T22:24:34.326 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create migration_log table" 2024-04-22T22:24:34.326 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create user table" 2024-04-22T22:24:34.326 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.login" 2024-04-22T22:24:34.326 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.email" 2024-04-22T22:24:34.327 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_login - v1" 2024-04-22T22:24:34.327 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_email - v1" 2024-04-22T22:24:34.327 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table user to user_v1 - v1" 2024-04-22T22:24:34.327 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create user table v2" 2024-04-22T22:24:34.327 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_login - v2" 2024-04-22T22:24:34.327 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_email - v2" 2024-04-22T22:24:34.327 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2024-04-22T22:24:34.327 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table user_v1" 2024-04-22T22:24:34.327 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add column help_flags1 to user table" 2024-04-22T22:24:34.327 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Update user table charset" 2024-04-22T22:24:34.327 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add last_seen_at column to user" 2024-04-22T22:24:34.327 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add missing user data" 2024-04-22T22:24:34.327 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_disabled column to user" 2024-04-22T22:24:34.327 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add index user.login/user.email" 2024-04-22T22:24:34.327 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_service_account column to user" 2024-04-22T22:24:34.328 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create temp user table v1-7" 2024-04-22T22:24:34.328 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v1-7" 2024-04-22T22:24:34.328 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v1-7" 2024-04-22T22:24:34.328 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v1-7" 2024-04-22T22:24:34.328 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v1-7" 2024-04-22T22:24:34.328 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Update temp_user table charset" 2024-04-22T22:24:34.328 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_email - v1" 2024-04-22T22:24:34.328 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_org_id - v1" 2024-04-22T22:24:34.328 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_code - v1" 2024-04-22T22:24:34.329 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_status - v1" 2024-04-22T22:24:34.329 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-04-22T22:24:34.329 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create temp_user v2" 2024-04-22T22:24:34.329 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v2" 2024-04-22T22:24:34.329 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v2" 2024-04-22T22:24:34.329 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v2" 2024-04-22T22:24:34.329 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v2" 2024-04-22T22:24:34.329 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="copy temp_user v1 to v2" 2024-04-22T22:24:34.329 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop temp_user_tmp_qwerty" 2024-04-22T22:24:34.329 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Set created for temp users that will otherwise prematurely expire" 2024-04-22T22:24:34.329 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create star table" 2024-04-22T22:24:34.329 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index star.user_id_dashboard_id" 2024-04-22T22:24:34.330 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create org table v1" 2024-04-22T22:24:34.330 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_name - v1" 2024-04-22T22:24:34.330 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create org_user table v1" 2024-04-22T22:24:34.330 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_org_id - v1" 2024-04-22T22:24:34.330 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_user_org_id_user_id - v1" 2024-04-22T22:24:34.330 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_user_id - v1" 2024-04-22T22:24:34.330 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Update org table charset" 2024-04-22T22:24:34.330 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Update org_user table charset" 2024-04-22T22:24:34.330 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate all Read Only Viewers to Viewers" 2024-04-22T22:24:34.330 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard table" 2024-04-22T22:24:34.330 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard.account_id" 2024-04-22T22:24:34.330 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_account_id_slug" 2024-04-22T22:24:34.330 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_tag table" 2024-04-22T22:24:34.330 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_tag.dasboard_id_term" 2024-04-22T22:24:34.331 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-04-22T22:24:34.331 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard to dashboard_v1 - v1" 2024-04-22T22:24:34.331 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard v2" 2024-04-22T22:24:34.331 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_org_id - v2" 2024-04-22T22:24:34.331 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 bash[38693]: 5d6c83f8480c9a452d14aab2194c813ff3d4be802405ce2ea435a5d459b911b3 2024-04-22T22:24:34.331 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_org_id_slug - v2" 2024-04-22T22:24:34.331 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard v1 to v2" 2024-04-22T22:24:34.331 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_v1" 2024-04-22T22:24:34.331 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 systemd[1]: Started Ceph grafana.a for 1ab1a3a8-00f6-11ef-bc93-c7b262605968. 2024-04-22T22:24:34.331 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard.data to mediumtext v1" 2024-04-22T22:24:34.331 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add column updated_by in dashboard - v2" 2024-04-22T22:24:34.331 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add column created_by in dashboard - v2" 2024-04-22T22:24:34.331 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add column gnetId in dashboard" 2024-04-22T22:24:34.331 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for gnetId in dashboard" 2024-04-22T22:24:34.331 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_id in dashboard" 2024-04-22T22:24:34.331 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for plugin_id in dashboard" 2024-04-22T22:24:34.332 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_id in dashboard_tag" 2024-04-22T22:24:34.332 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard table charset" 2024-04-22T22:24:34.332 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_tag table charset" 2024-04-22T22:24:34.332 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add column folder_id in dashboard" 2024-04-22T22:24:34.332 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add column isFolder in dashboard" 2024-04-22T22:24:34.332 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add column has_acl in dashboard" 2024-04-22T22:24:34.332 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in dashboard" 2024-04-22T22:24:34.332 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in dashboard" 2024-04-22T22:24:34.332 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index dashboard_org_id_uid" 2024-04-22T22:24:34.332 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_slug" 2024-04-22T22:24:34.332 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard title length" 2024-04-22T22:24:34.333 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index for dashboard_org_id_title_folder_id" 2024-04-22T22:24:34.333 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning" 2024-04-22T22:24:34.333 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-04-22T22:24:34.333 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning v2" 2024-04-22T22:24:34.333 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-04-22T22:24:34.333 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-04-22T22:24:34.333 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard_provisioning v1 to v2" 2024-04-22T22:24:34.333 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop dashboard_provisioning_tmp_qwerty" 2024-04-22T22:24:34.333 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add check_sum column" 2024-04-22T22:24:34.333 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_title" 2024-04-22T22:24:34.333 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="delete tags for deleted dashboards" 2024-04-22T22:24:34.333 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="delete stars for deleted dashboards" 2024-04-22T22:24:34.334 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_is_folder" 2024-04-22T22:24:34.334 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table" 2024-04-22T22:24:34.334 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index data_source.account_id" 2024-04-22T22:24:34.334 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index data_source.account_id_name" 2024-04-22T22:24:34.334 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_data_source_account_id - v1" 2024-04-22T22:24:34.334 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_data_source_account_id_name - v1" 2024-04-22T22:24:34.334 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table data_source to data_source_v1 - v1" 2024-04-22T22:24:34.334 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table v2" 2024-04-22T22:24:34.334 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_data_source_org_id - v2" 2024-04-22T22:24:34.334 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_data_source_org_id_name - v2" 2024-04-22T22:24:34.334 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2024-04-22T22:24:34.334 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table data_source_v1 #2" 2024-04-22T22:24:34.575 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add column with_credentials" 2024-04-22T22:24:34.575 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add secure json data column" 2024-04-22T22:24:34.575 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Update data_source table charset" 2024-04-22T22:24:34.575 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Update initial version to 1" 2024-04-22T22:24:34.575 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add read_only data column" 2024-04-22T22:24:34.575 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate logging ds to loki ds" 2024-04-22T22:24:34.576 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Update json_data with nulls" 2024-04-22T22:24:34.576 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add uid column" 2024-04-22T22:24:34.576 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid value" 2024-04-22T22:24:34.576 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index datasource_org_id_uid" 2024-04-22T22:24:34.576 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index datasource_org_id_is_default" 2024-04-22T22:24:34.576 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table" 2024-04-22T22:24:34.576 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id" 2024-04-22T22:24:34.576 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.key" 2024-04-22T22:24:34.576 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id_name" 2024-04-22T22:24:34.576 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_api_key_account_id - v1" 2024-04-22T22:24:34.576 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_key - v1" 2024-04-22T22:24:34.576 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_account_id_name - v1" 2024-04-22T22:24:34.577 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table api_key to api_key_v1 - v1" 2024-04-22T22:24:34.577 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table v2" 2024-04-22T22:24:34.578 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_api_key_org_id - v2" 2024-04-22T22:24:34.578 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_key - v2" 2024-04-22T22:24:34.578 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_org_id_name - v2" 2024-04-22T22:24:34.578 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="copy api_key v1 to v2" 2024-04-22T22:24:34.578 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table api_key_v1" 2024-04-22T22:24:34.578 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Update api_key table charset" 2024-04-22T22:24:34.578 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add expires to api_key table" 2024-04-22T22:24:34.578 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add service account foreign key" 2024-04-22T22:24:34.578 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v4" 2024-04-22T22:24:34.579 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_snapshot_v4 #1" 2024-04-22T22:24:34.579 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v5 #2" 2024-04-22T22:24:34.579 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_key - v5" 2024-04-22T22:24:34.579 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-04-22T22:24:34.579 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_snapshot_user_id - v5" 2024-04-22T22:24:34.579 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_snapshot to mediumtext v2" 2024-04-22T22:24:34.579 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_snapshot table charset" 2024-04-22T22:24:34.579 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external_delete_url to dashboard_snapshots table" 2024-04-22T22:24:34.579 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add encrypted dashboard json column" 2024-04-22T22:24:34.579 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-04-22T22:24:34.579 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create quota table v1" 2024-04-22T22:24:34.579 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_quota_org_id_user_id_target - v1" 2024-04-22T22:24:34.579 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Update quota table charset" 2024-04-22T22:24:34.579 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create plugin_setting table" 2024-04-22T22:24:34.580 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-04-22T22:24:34.580 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_version to plugin_settings" 2024-04-22T22:24:34.580 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Update plugin_setting table charset" 2024-04-22T22:24:34.580 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create session table" 2024-04-22T22:24:34.580 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist table" 2024-04-22T22:24:34.580 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist_item table" 2024-04-22T22:24:34.580 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist table v2" 2024-04-22T22:24:34.580 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist item table v2" 2024-04-22T22:24:34.580 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist table charset" 2024-04-22T22:24:34.580 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist_item table charset" 2024-04-22T22:24:34.580 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v2" 2024-04-22T22:24:34.580 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v3" 2024-04-22T22:24:34.580 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create preferences table v3" 2024-04-22T22:24:34.580 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Update preferences table charset" 2024-04-22T22:24:34.580 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add column team_id in preferences" 2024-04-22T22:24:34.581 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Update team_id column values in preferences" 2024-04-22T22:24:34.581 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add column week_start in preferences" 2024-04-22T22:24:34.581 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create alert table v1" 2024-04-22T22:24:34.581 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert org_id & id " 2024-04-22T22:24:34.581 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert state" 2024-04-22T22:24:34.581 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert dashboard_id" 2024-04-22T22:24:34.581 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v1" 2024-04-22T22:24:34.581 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-04-22T22:24:34.581 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-04-22T22:24:34.581 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-04-22T22:24:34.581 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v2" 2024-04-22T22:24:34.581 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-04-22T22:24:34.581 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="copy alert_rule_tag v1 to v2" 2024-04-22T22:24:34.581 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop table alert_rule_tag_v1" 2024-04-22T22:24:34.581 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification table v1" 2024-04-22T22:24:34.582 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add column is_default" 2024-04-22T22:24:34.582 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2024-04-22T22:24:34.582 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2024-04-22T22:24:34.582 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2024-04-22T22:24:34.582 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2024-04-22T22:24:34.582 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2024-04-22T22:24:34.582 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2024-04-22T22:24:34.582 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2024-04-22T22:24:34.582 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2024-04-22T22:24:34.582 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2024-04-22T22:24:34.582 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2024-04-22T22:24:34.582 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-04-22T22:24:34.582 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2024-04-22T22:24:34.582 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2024-04-22T22:24:34.583 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2024-04-22T22:24:34.583 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2024-04-22T22:24:34.583 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2024-04-22T22:24:34.583 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add column secure_settings in alert_notification" 2024-04-22T22:24:34.583 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert.settings to mediumtext" 2024-04-22T22:24:34.583 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_notification_state_alert_id" 2024-04-22T22:24:34.583 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_rule_tag_alert_id" 2024-04-22T22:24:34.583 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2024-04-22T22:24:34.583 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2024-04-22T22:24:34.583 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2024-04-22T22:24:34.583 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2024-04-22T22:24:34.583 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2024-04-22T22:24:34.583 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2024-04-22T22:24:34.583 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2024-04-22T22:24:34.583 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2024-04-22T22:24:34.584 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2024-04-22T22:24:34.584 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2024-04-22T22:24:34.584 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2024-04-22T22:24:34.584 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2024-04-22T22:24:34.584 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2024-04-22T22:24:34.584 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-04-22T22:24:34.584 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-04-22T22:24:34.584 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v3" 2024-04-22T22:24:34.584 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-04-22T22:24:34.584 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="copy annotation_tag v2 to v3" 2024-04-22T22:24:34.584 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop table annotation_tag_v2" 2024-04-22T22:24:34.584 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2024-04-22T22:24:34.584 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2024-04-22T22:24:34.584 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2024-04-22T22:24:34.585 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2024-04-22T22:24:34.585 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2024-04-22T22:24:34.585 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2024-04-22T22:24:34.585 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2024-04-22T22:24:34.585 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2024-04-22T22:24:34.585 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2024-04-22T22:24:34.585 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2024-04-22T22:24:34.585 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2024-04-22T22:24:34.585 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-04-22T22:24:34.585 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-04-22T22:24:34.585 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2024-04-22T22:24:34.585 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2024-04-22T22:24:34.585 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2024-04-22T22:24:34.585 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2024-04-22T22:24:34.585 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2024-04-22T22:24:34.586 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2024-04-22T22:24:34.586 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-04-22T22:24:34.586 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2024-04-22T22:24:34.586 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2024-04-22T22:24:34.586 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2024-04-22T22:24:34.586 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2024-04-22T22:24:34.586 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2024-04-22T22:24:34.586 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2024-04-22T22:24:34.586 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2024-04-22T22:24:34.586 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2024-04-22T22:24:34.586 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2024-04-22T22:24:34.586 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.team_id" 2024-04-22T22:24:34.586 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2024-04-22T22:24:34.586 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2024-04-22T22:24:34.586 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2024-04-22T22:24:34.587 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2024-04-22T22:24:34.587 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2024-04-22T22:24:34.587 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2024-04-22T22:24:34.587 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2024-04-22T22:24:34.587 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_user_id" 2024-04-22T22:24:34.587 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_team_id" 2024-04-22T22:24:34.587 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_org_id_role" 2024-04-22T22:24:34.587 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_permission" 2024-04-22T22:24:34.587 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2024-04-22T22:24:34.587 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="delete acl rules for deleted dashboards and folders" 2024-04-22T22:24:34.587 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2024-04-22T22:24:34.587 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2024-04-22T22:24:34.587 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2024-04-22T22:24:34.588 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2024-04-22T22:24:34.588 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2024-04-22T22:24:34.588 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-04-22T22:24:34.588 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2024-04-22T22:24:34.588 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2024-04-22T22:24:34.588 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2024-04-22T22:24:34.588 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2024-04-22T22:24:34.588 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2024-04-22T22:24:34.588 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-04-22T22:24:34.588 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2024-04-22T22:24:34.588 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2024-04-22T22:24:34.589 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2024-04-22T22:24:34.589 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2024-04-22T22:24:34.589 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2024-04-22T22:24:34.589 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2024-04-22T22:24:34.589 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2024-04-22T22:24:34.589 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2024-04-22T22:24:34.589 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2024-04-22T22:24:34.589 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2024-04-22T22:24:34.589 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2024-04-22T22:24:34.589 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_auth_token.user_id" 2024-04-22T22:24:34.589 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add revoked_at to the user auth token" 2024-04-22T22:24:34.589 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2024-04-22T22:24:35.007 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-mon[22689]: pgmap v43: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:35.007 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:35.007 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:24:35.007 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:24:35.007 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2024-04-22T22:24:35.008 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create short_url table v1" 2024-04-22T22:24:35.008 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index short_url.org_id-uid" 2024-04-22T22:24:35.008 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition table" 2024-04-22T22:24:35.008 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition table" 2024-04-22T22:24:35.008 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and title columns" 2024-04-22T22:24:35.008 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and uid columns" 2024-04-22T22:24:35.008 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition table data column to mediumtext in mysql" 2024-04-22T22:24:35.008 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and title columns" 2024-04-22T22:24:35.008 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and uid columns" 2024-04-22T22:24:35.008 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and title columns" 2024-04-22T22:24:35.008 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and uid columns" 2024-04-22T22:24:35.008 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add column paused in alert_definition" 2024-04-22T22:24:35.008 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition table" 2024-04-22T22:24:35.009 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition_version table" 2024-04-22T22:24:35.009 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition_version table" 2024-04-22T22:24:35.009 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-04-22T22:24:35.009 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-04-22T22:24:35.009 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition_version table data column to mediumtext in mysql" 2024-04-22T22:24:35.009 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition_version table" 2024-04-22T22:24:35.009 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_instance table" 2024-04-22T22:24:35.009 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-04-22T22:24:35.009 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, current_state columns" 2024-04-22T22:24:35.009 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add column current_state_end to alert_instance" 2024-04-22T22:24:35.009 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-04-22T22:24:35.009 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, current_state on alert_instance" 2024-04-22T22:24:35.009 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_org_id to rule_org_id in alert_instance" 2024-04-22T22:24:35.009 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_uid to rule_uid in alert_instance" 2024-04-22T22:24:35.009 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-04-22T22:24:35.010 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, current_state on alert_instance" 2024-04-22T22:24:35.010 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule table" 2024-04-22T22:24:35.010 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and title columns" 2024-04-22T22:24:35.010 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and uid columns" 2024-04-22T22:24:35.010 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-04-22T22:24:35.010 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule table data column to mediumtext in mysql" 2024-04-22T22:24:35.010 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule" 2024-04-22T22:24:35.010 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule" 2024-04-22T22:24:35.011 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:34 smithi067 ceph-mon[21875]: pgmap v43: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:35.011 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:34 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:35.011 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:34 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:24:35.011 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:34 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:24:35.011 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:34 smithi067 ceph-mon[26633]: pgmap v43: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:35.011 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:34 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:35.011 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:34 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:24:35.011 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:34 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:24:35.012 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule" 2024-04-22T22:24:35.012 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="remove unique index from alert_rule on org_id, title columns" 2024-04-22T22:24:35.012 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-04-22T22:24:35.012 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add dashboard_uid column to alert_rule" 2024-04-22T22:24:35.012 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add panel_id column to alert_rule" 2024-04-22T22:24:35.012 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-04-22T22:24:35.012 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule_version table" 2024-04-22T22:24:35.012 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-04-22T22:24:35.012 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-04-22T22:24:35.012 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule_version table data column to mediumtext in mysql" 2024-04-22T22:24:35.012 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule_version" 2024-04-22T22:24:35.012 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule_version" 2024-04-22T22:24:35.012 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule_version" 2024-04-22T22:24:35.012 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id=create_alert_configuration_table 2024-04-22T22:24:35.013 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add column default in alert_configuration" 2024-04-22T22:24:35.013 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-04-22T22:24:35.013 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add column org_id in alert_configuration" 2024-04-22T22:24:35.013 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_configuration table on org_id column" 2024-04-22T22:24:35.013 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id=create_ngalert_configuration_table 2024-04-22T22:24:35.013 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index in ngalert_configuration on org_id column" 2024-04-22T22:24:35.013 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="clear migration entry \"remove unified alerting data\"" 2024-04-22T22:24:35.013 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="move dashboard alerts to unified alerting" 2024-04-22T22:24:35.013 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element table v1" 2024-04-22T22:24:35.013 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element org_id-folder_id-name-kind" 2024-04-22T22:24:35.013 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element_connection table v1" 2024-04-22T22:24:35.013 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element_connection element_id-kind-connection_id" 2024-04-22T22:24:35.013 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index library_element org_id_uid" 2024-04-22T22:24:35.013 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="clone move dashboard alerts to unified alerting" 2024-04-22T22:24:35.013 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create data_keys table" 2024-04-22T22:24:35.013 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create kv_store table v1" 2024-04-22T22:24:35.014 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index kv_store.org_id-namespace-key" 2024-04-22T22:24:35.014 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="update dashboard_uid and panel_id from existing annotations" 2024-04-22T22:24:35.014 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create permission table" 2024-04-22T22:24:35.014 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index permission.role_id" 2024-04-22T22:24:35.014 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_id_action_scope" 2024-04-22T22:24:35.014 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create role table" 2024-04-22T22:24:35.014 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add column display_name" 2024-04-22T22:24:35.014 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add column group_name" 2024-04-22T22:24:35.014 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index role.org_id" 2024-04-22T22:24:35.014 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_org_id_name" 2024-04-22T22:24:35.014 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index role_org_id_uid" 2024-04-22T22:24:35.014 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create team role table" 2024-04-22T22:24:35.014 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.org_id" 2024-04-22T22:24:35.014 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_role_org_id_team_id_role_id" 2024-04-22T22:24:35.014 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.team_id" 2024-04-22T22:24:35.015 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create user role table" 2024-04-22T22:24:35.015 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.org_id" 2024-04-22T22:24:35.015 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_role_org_id_user_id_role_id" 2024-04-22T22:24:35.015 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.user_id" 2024-04-22T22:24:35.015 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create builtin role table" 2024-04-22T22:24:35.015 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.role_id" 2024-04-22T22:24:35.015 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.name" 2024-04-22T22:24:35.015 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Add column org_id to builtin_role table" 2024-04-22T22:24:35.015 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.org_id" 2024-04-22T22:24:35.015 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_org_id_role_id_role" 2024-04-22T22:24:35.015 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index role_org_id_uid" 2024-04-22T22:24:35.015 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role.uid" 2024-04-22T22:24:35.015 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="create seed assignment table" 2024-04-22T22:24:35.015 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_role_name" 2024-04-22T22:24:35.015 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="migrations completed" logger=migrator performed=381 skipped=0 duration=470.338787ms 2024-04-22T22:24:35.015 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Created default organization" logger=sqlstore 2024-04-22T22:24:35.015 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2024-04-22T22:24:35.016 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2024-04-22T22:24:35.016 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2024-04-22T22:24:35.016 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2024-04-22T22:24:35.016 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2024-04-22T22:24:35.016 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-04-22T22:24:35.016 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2024-04-22T22:24:35.016 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="warming cache for startup" logger=ngalert 2024-04-22T22:24:35.016 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-04-22T22:24:35.016 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:24:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:24:34+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2024-04-22T22:24:35.847 INFO:teuthology.orchestra.run.smithi067.stdout:{ 2024-04-22T22:24:35.847 INFO:teuthology.orchestra.run.smithi067.stdout: "id": "5415be30-74e7-47c2-9bf7-2ba8315931bf", 2024-04-22T22:24:35.847 INFO:teuthology.orchestra.run.smithi067.stdout: "name": "r", 2024-04-22T22:24:35.847 INFO:teuthology.orchestra.run.smithi067.stdout: "current_period": "a73e1e36-577c-423f-8623-0b51247ebf7f", 2024-04-22T22:24:35.848 INFO:teuthology.orchestra.run.smithi067.stdout: "epoch": 1 2024-04-22T22:24:35.848 INFO:teuthology.orchestra.run.smithi067.stdout:} 2024-04-22T22:24:36.000 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:35 smithi086 ceph-mon[22689]: osdmap e61: 8 total, 8 up, 8 in 2024-04-22T22:24:36.000 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:35 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/1527900043' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-22T22:24:36.000 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:35 smithi086 ceph-mon[22689]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-22T22:24:36.149 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:35 smithi067 ceph-mon[26633]: osdmap e61: 8 total, 8 up, 8 in 2024-04-22T22:24:36.149 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:35 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/1527900043' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-22T22:24:36.150 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:35 smithi067 ceph-mon[26633]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-22T22:24:36.150 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:35 smithi067 ceph-mon[21875]: osdmap e61: 8 total, 8 up, 8 in 2024-04-22T22:24:36.150 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:35 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/1527900043' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-22T22:24:36.150 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:35 smithi067 ceph-mon[21875]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-22T22:24:36.490 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -e sha1=430e09df97c8fc7dc2b2ae424f68ed11366c540f -- bash -c 'radosgw-admin zonegroup create --rgw-zonegroup=default --master --default' 2024-04-22T22:24:37.105 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:36 smithi086 ceph-mon[22689]: pgmap v45: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:37.105 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:36 smithi086 ceph-mon[22689]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-04-22T22:24:37.105 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:36 smithi086 ceph-mon[22689]: osdmap e62: 8 total, 8 up, 8 in 2024-04-22T22:24:37.258 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:36 smithi067 ceph-mon[21875]: pgmap v45: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:37.258 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:36 smithi067 ceph-mon[21875]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-04-22T22:24:37.258 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:36 smithi067 ceph-mon[21875]: osdmap e62: 8 total, 8 up, 8 in 2024-04-22T22:24:37.258 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:36 smithi067 ceph-mon[26633]: pgmap v45: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:37.258 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:36 smithi067 ceph-mon[26633]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-04-22T22:24:37.258 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:36 smithi067 ceph-mon[26633]: osdmap e62: 8 total, 8 up, 8 in 2024-04-22T22:24:37.981 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:37 smithi086 ceph-mon[22689]: osdmap e63: 8 total, 8 up, 8 in 2024-04-22T22:24:38.008 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:37 smithi067 ceph-mon[21875]: osdmap e63: 8 total, 8 up, 8 in 2024-04-22T22:24:38.008 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:37 smithi067 ceph-mon[26633]: osdmap e63: 8 total, 8 up, 8 in 2024-04-22T22:24:39.225 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:39 smithi067 ceph-mon[21875]: pgmap v48: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:39.225 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:39 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:39.226 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:39 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:39.226 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:39 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:39.226 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:39 smithi067 ceph-mon[26633]: pgmap v48: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:39.226 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:39 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:39.226 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:39 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:39.226 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:39 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:39.256 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:39 smithi086 ceph-mon[22689]: pgmap v48: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:39.257 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:39 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:39.257 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:39 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:39.257 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:39 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:39.341 INFO:teuthology.orchestra.run.smithi067.stdout:{ 2024-04-22T22:24:39.341 INFO:teuthology.orchestra.run.smithi067.stdout: "id": "115d9175-dd64-4999-913d-15472a1de9cd", 2024-04-22T22:24:39.341 INFO:teuthology.orchestra.run.smithi067.stdout: "name": "default", 2024-04-22T22:24:39.341 INFO:teuthology.orchestra.run.smithi067.stdout: "api_name": "default", 2024-04-22T22:24:39.341 INFO:teuthology.orchestra.run.smithi067.stdout: "is_master": "true", 2024-04-22T22:24:39.341 INFO:teuthology.orchestra.run.smithi067.stdout: "endpoints": [], 2024-04-22T22:24:39.341 INFO:teuthology.orchestra.run.smithi067.stdout: "hostnames": [], 2024-04-22T22:24:39.341 INFO:teuthology.orchestra.run.smithi067.stdout: "hostnames_s3website": [], 2024-04-22T22:24:39.341 INFO:teuthology.orchestra.run.smithi067.stdout: "master_zone": "", 2024-04-22T22:24:39.341 INFO:teuthology.orchestra.run.smithi067.stdout: "zones": [], 2024-04-22T22:24:39.341 INFO:teuthology.orchestra.run.smithi067.stdout: "placement_targets": [], 2024-04-22T22:24:39.342 INFO:teuthology.orchestra.run.smithi067.stdout: "default_placement": "", 2024-04-22T22:24:39.342 INFO:teuthology.orchestra.run.smithi067.stdout: "realm_id": "5415be30-74e7-47c2-9bf7-2ba8315931bf", 2024-04-22T22:24:39.342 INFO:teuthology.orchestra.run.smithi067.stdout: "sync_policy": { 2024-04-22T22:24:39.342 INFO:teuthology.orchestra.run.smithi067.stdout: "groups": [] 2024-04-22T22:24:39.342 INFO:teuthology.orchestra.run.smithi067.stdout: } 2024-04-22T22:24:39.342 INFO:teuthology.orchestra.run.smithi067.stdout:} 2024-04-22T22:24:39.880 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -e sha1=430e09df97c8fc7dc2b2ae424f68ed11366c540f -- bash -c 'radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default' 2024-04-22T22:24:40.758 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:24:40 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:24:40] "GET /metrics HTTP/1.1" 200 191483 "" "Prometheus/2.33.4" 2024-04-22T22:24:41.148 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:40 smithi067 ceph-mon[21875]: pgmap v49: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 1.8 KiB/s wr, 2 op/s 2024-04-22T22:24:41.149 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:40 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:41.149 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:40 smithi067 ceph-mon[21875]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-22T22:24:41.149 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:40 smithi067 ceph-mon[21875]: Reconfiguring daemon alertmanager.a on smithi067 2024-04-22T22:24:41.149 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:40 smithi067 ceph-mon[26633]: pgmap v49: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 1.8 KiB/s wr, 2 op/s 2024-04-22T22:24:41.149 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:40 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:41.149 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:40 smithi067 ceph-mon[26633]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-22T22:24:41.149 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:40 smithi067 ceph-mon[26633]: Reconfiguring daemon alertmanager.a on smithi067 2024-04-22T22:24:41.257 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:40 smithi086 ceph-mon[22689]: pgmap v49: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 682 B/s rd, 1.8 KiB/s wr, 2 op/s 2024-04-22T22:24:41.257 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:40 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:41.257 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:40 smithi086 ceph-mon[22689]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-22T22:24:41.257 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:40 smithi086 ceph-mon[22689]: Reconfiguring daemon alertmanager.a on smithi067 2024-04-22T22:24:41.757 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:24:41 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:24:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:24:43.240 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:42 smithi067 ceph-mon[21875]: pgmap v50: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 626 B/s rd, 1.7 KiB/s wr, 2 op/s 2024-04-22T22:24:43.241 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:42 smithi067 ceph-mon[26633]: pgmap v50: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 626 B/s rd, 1.7 KiB/s wr, 2 op/s 2024-04-22T22:24:43.256 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:42 smithi086 ceph-mon[22689]: pgmap v50: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 626 B/s rd, 1.7 KiB/s wr, 2 op/s 2024-04-22T22:24:43.767 INFO:teuthology.orchestra.run.smithi067.stdout:{ 2024-04-22T22:24:43.767 INFO:teuthology.orchestra.run.smithi067.stdout: "id": "8a3bce13-588b-405a-b6eb-772a44cd8dfb", 2024-04-22T22:24:43.767 INFO:teuthology.orchestra.run.smithi067.stdout: "name": "z", 2024-04-22T22:24:43.767 INFO:teuthology.orchestra.run.smithi067.stdout: "domain_root": "z.rgw.meta:root", 2024-04-22T22:24:43.767 INFO:teuthology.orchestra.run.smithi067.stdout: "control_pool": "z.rgw.control", 2024-04-22T22:24:43.767 INFO:teuthology.orchestra.run.smithi067.stdout: "gc_pool": "z.rgw.log:gc", 2024-04-22T22:24:43.767 INFO:teuthology.orchestra.run.smithi067.stdout: "lc_pool": "z.rgw.log:lc", 2024-04-22T22:24:43.767 INFO:teuthology.orchestra.run.smithi067.stdout: "log_pool": "z.rgw.log", 2024-04-22T22:24:43.767 INFO:teuthology.orchestra.run.smithi067.stdout: "intent_log_pool": "z.rgw.log:intent", 2024-04-22T22:24:43.767 INFO:teuthology.orchestra.run.smithi067.stdout: "usage_log_pool": "z.rgw.log:usage", 2024-04-22T22:24:43.767 INFO:teuthology.orchestra.run.smithi067.stdout: "roles_pool": "z.rgw.meta:roles", 2024-04-22T22:24:43.768 INFO:teuthology.orchestra.run.smithi067.stdout: "reshard_pool": "z.rgw.log:reshard", 2024-04-22T22:24:43.768 INFO:teuthology.orchestra.run.smithi067.stdout: "user_keys_pool": "z.rgw.meta:users.keys", 2024-04-22T22:24:43.768 INFO:teuthology.orchestra.run.smithi067.stdout: "user_email_pool": "z.rgw.meta:users.email", 2024-04-22T22:24:43.768 INFO:teuthology.orchestra.run.smithi067.stdout: "user_swift_pool": "z.rgw.meta:users.swift", 2024-04-22T22:24:43.768 INFO:teuthology.orchestra.run.smithi067.stdout: "user_uid_pool": "z.rgw.meta:users.uid", 2024-04-22T22:24:43.768 INFO:teuthology.orchestra.run.smithi067.stdout: "otp_pool": "z.rgw.otp", 2024-04-22T22:24:43.768 INFO:teuthology.orchestra.run.smithi067.stdout: "system_key": { 2024-04-22T22:24:43.768 INFO:teuthology.orchestra.run.smithi067.stdout: "access_key": "", 2024-04-22T22:24:43.768 INFO:teuthology.orchestra.run.smithi067.stdout: "secret_key": "" 2024-04-22T22:24:43.768 INFO:teuthology.orchestra.run.smithi067.stdout: }, 2024-04-22T22:24:43.768 INFO:teuthology.orchestra.run.smithi067.stdout: "placement_pools": [ 2024-04-22T22:24:43.768 INFO:teuthology.orchestra.run.smithi067.stdout: { 2024-04-22T22:24:43.768 INFO:teuthology.orchestra.run.smithi067.stdout: "key": "default-placement", 2024-04-22T22:24:43.768 INFO:teuthology.orchestra.run.smithi067.stdout: "val": { 2024-04-22T22:24:43.768 INFO:teuthology.orchestra.run.smithi067.stdout: "index_pool": "z.rgw.buckets.index", 2024-04-22T22:24:43.769 INFO:teuthology.orchestra.run.smithi067.stdout: "storage_classes": { 2024-04-22T22:24:43.769 INFO:teuthology.orchestra.run.smithi067.stdout: "STANDARD": { 2024-04-22T22:24:43.769 INFO:teuthology.orchestra.run.smithi067.stdout: "data_pool": "z.rgw.buckets.data" 2024-04-22T22:24:43.769 INFO:teuthology.orchestra.run.smithi067.stdout: } 2024-04-22T22:24:43.769 INFO:teuthology.orchestra.run.smithi067.stdout: }, 2024-04-22T22:24:43.769 INFO:teuthology.orchestra.run.smithi067.stdout: "data_extra_pool": "z.rgw.buckets.non-ec", 2024-04-22T22:24:43.769 INFO:teuthology.orchestra.run.smithi067.stdout: "index_type": 0 2024-04-22T22:24:43.769 INFO:teuthology.orchestra.run.smithi067.stdout: } 2024-04-22T22:24:43.769 INFO:teuthology.orchestra.run.smithi067.stdout: } 2024-04-22T22:24:43.769 INFO:teuthology.orchestra.run.smithi067.stdout: ], 2024-04-22T22:24:43.769 INFO:teuthology.orchestra.run.smithi067.stdout: "realm_id": "5415be30-74e7-47c2-9bf7-2ba8315931bf", 2024-04-22T22:24:43.769 INFO:teuthology.orchestra.run.smithi067.stdout: "notif_pool": "z.rgw.log:notif" 2024-04-22T22:24:43.769 INFO:teuthology.orchestra.run.smithi067.stdout:} 2024-04-22T22:24:44.420 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -e sha1=430e09df97c8fc7dc2b2ae424f68ed11366c540f -- bash -c 'radosgw-admin period update --rgw-realm=r --commit' 2024-04-22T22:24:45.083 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:44 smithi067 ceph-mon[21875]: pgmap v51: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 1.4 KiB/s wr, 2 op/s 2024-04-22T22:24:45.084 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:44 smithi067 ceph-mon[26633]: pgmap v51: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 1.4 KiB/s wr, 2 op/s 2024-04-22T22:24:45.084 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:44 smithi067 systemd[1]: Stopping Ceph alertmanager.a for 1ab1a3a8-00f6-11ef-bc93-c7b262605968... 2024-04-22T22:24:45.084 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:44 smithi067 bash[46474]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager.a" found: no such container 2024-04-22T22:24:45.257 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:44 smithi086 ceph-mon[22689]: pgmap v51: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 1.4 KiB/s wr, 2 op/s 2024-04-22T22:24:45.420 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:45 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager-a[44785]: level=info ts=2024-04-22T22:24:45.306Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-04-22T22:24:46.008 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:45 smithi067 podman[46491]: 2024-04-22 22:24:45.76603674 +0000 UTC m=+0.765032103 container died cabfcc88c704345c6aa854bd1eb602c1fd87e04e2d98679d8d9c68fdbb56b711 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T22:24:46.383 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:46 smithi067 podman[46491]: 2024-04-22 22:24:46.216877611 +0000 UTC m=+1.215872987 container cleanup cabfcc88c704345c6aa854bd1eb602c1fd87e04e2d98679d8d9c68fdbb56b711 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T22:24:46.383 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:46 smithi067 bash[46491]: ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager-a 2024-04-22T22:24:46.695 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:46 smithi067 podman[46510]: 2024-04-22 22:24:46.392959529 +0000 UTC m=+1.053377670 container remove cabfcc88c704345c6aa854bd1eb602c1fd87e04e2d98679d8d9c68fdbb56b711 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T22:24:46.695 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:46 smithi067 podman[46510]: 2024-04-22 22:24:46.493765788 +0000 UTC m=+1.154183928 volume remove f42f03ef11640a2ad9f062167873fb06797c361f7648a958cc90755ee7bb32d0 2024-04-22T22:24:46.695 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:46 smithi067 bash[46536]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager.a" found: no such container 2024-04-22T22:24:46.695 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:46 smithi067 systemd[1]: ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-22T22:24:46.695 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:46 smithi067 systemd[1]: Stopped Ceph alertmanager.a for 1ab1a3a8-00f6-11ef-bc93-c7b262605968. 2024-04-22T22:24:46.696 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:46 smithi067 systemd[1]: Starting Ceph alertmanager.a for 1ab1a3a8-00f6-11ef-bc93-c7b262605968... 2024-04-22T22:24:47.257 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:46 smithi086 ceph-mon[22689]: pgmap v52: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 752 B/s rd, 1.7 KiB/s wr, 2 op/s 2024-04-22T22:24:47.258 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:46 smithi067 ceph-mon[21875]: pgmap v52: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 752 B/s rd, 1.7 KiB/s wr, 2 op/s 2024-04-22T22:24:47.258 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:46 smithi067 ceph-mon[26633]: pgmap v52: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 752 B/s rd, 1.7 KiB/s wr, 2 op/s 2024-04-22T22:24:48.006 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:47 smithi067 podman[46685]: 2024-04-22 22:24:47.724861779 +0000 UTC m=+0.080087038 volume create 2dc21b77266e00edcde2c70c7b8a63a66aecda4229b642c5796e84d02a8715fa 2024-04-22T22:24:48.006 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:47 smithi067 podman[46685]: 2024-04-22 22:24:47.666286076 +0000 UTC m=+0.021511344 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-04-22T22:24:48.006 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:47 smithi067 podman[46685]: 2024-04-22 22:24:47.841981884 +0000 UTC m=+0.197207191 container create 9d0e1f18c1cbfb68a5b722204cdfa86350fc0ef104a7cbf7b747d20e8973de8c (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T22:24:48.508 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:48 smithi067 podman[46685]: 2024-04-22 22:24:48.136173807 +0000 UTC m=+0.491399062 container init 9d0e1f18c1cbfb68a5b722204cdfa86350fc0ef104a7cbf7b747d20e8973de8c (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T22:24:48.509 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:48 smithi067 podman[46685]: 2024-04-22 22:24:48.138709422 +0000 UTC m=+0.493934677 container start 9d0e1f18c1cbfb68a5b722204cdfa86350fc0ef104a7cbf7b747d20e8973de8c (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T22:24:48.509 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:48 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager-a[46701]: level=info ts=2024-04-22T22:24:48.149Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-04-22T22:24:48.509 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:48 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager-a[46701]: level=info ts=2024-04-22T22:24:48.149Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-04-22T22:24:48.509 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:48 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager-a[46701]: level=info ts=2024-04-22T22:24:48.150Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.67 port=9094 2024-04-22T22:24:48.509 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:48 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager-a[46701]: level=info ts=2024-04-22T22:24:48.151Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-22T22:24:48.509 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:48 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager-a[46701]: level=info ts=2024-04-22T22:24:48.177Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-22T22:24:48.509 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:48 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager-a[46701]: level=info ts=2024-04-22T22:24:48.177Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-22T22:24:48.509 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:48 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager-a[46701]: level=info ts=2024-04-22T22:24:48.179Z caller=main.go:518 msg=Listening address=:9093 2024-04-22T22:24:48.509 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:48 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager-a[46701]: level=info ts=2024-04-22T22:24:48.179Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-04-22T22:24:48.509 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:48 smithi067 bash[46685]: 9d0e1f18c1cbfb68a5b722204cdfa86350fc0ef104a7cbf7b747d20e8973de8c 2024-04-22T22:24:48.509 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:48 smithi067 systemd[1]: Started Ceph alertmanager.a for 1ab1a3a8-00f6-11ef-bc93-c7b262605968. 2024-04-22T22:24:49.257 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:49 smithi086 ceph-mon[22689]: pgmap v53: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 680 B/s rd, 1.5 KiB/s wr, 2 op/s 2024-04-22T22:24:49.257 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:49 smithi086 ceph-mon[22689]: osdmap e64: 8 total, 8 up, 8 in 2024-04-22T22:24:49.257 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:49 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/2422096363' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-22T22:24:49.257 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:49 smithi086 ceph-mon[22689]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-22T22:24:49.257 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:49 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:49.257 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:49 smithi086 ceph-mon[22689]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-22T22:24:49.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:49 smithi086 ceph-mon[22689]: Reconfiguring daemon prometheus.a on smithi086 2024-04-22T22:24:49.258 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:49 smithi067 ceph-mon[21875]: pgmap v53: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 680 B/s rd, 1.5 KiB/s wr, 2 op/s 2024-04-22T22:24:49.258 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:49 smithi067 ceph-mon[21875]: osdmap e64: 8 total, 8 up, 8 in 2024-04-22T22:24:49.258 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:49 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/2422096363' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-22T22:24:49.258 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:49 smithi067 ceph-mon[21875]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-22T22:24:49.258 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:49 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:49.258 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:49 smithi067 ceph-mon[21875]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-22T22:24:49.258 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:49 smithi067 ceph-mon[21875]: Reconfiguring daemon prometheus.a on smithi086 2024-04-22T22:24:49.259 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:49 smithi067 ceph-mon[26633]: pgmap v53: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 680 B/s rd, 1.5 KiB/s wr, 2 op/s 2024-04-22T22:24:49.259 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:49 smithi067 ceph-mon[26633]: osdmap e64: 8 total, 8 up, 8 in 2024-04-22T22:24:49.259 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:49 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/2422096363' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-22T22:24:49.259 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:49 smithi067 ceph-mon[26633]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-22T22:24:49.259 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:49 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:49.259 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:49 smithi067 ceph-mon[26633]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-22T22:24:49.259 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:49 smithi067 ceph-mon[26633]: Reconfiguring daemon prometheus.a on smithi086 2024-04-22T22:24:50.257 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:50 smithi086 ceph-mon[22689]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-04-22T22:24:50.257 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:50 smithi086 ceph-mon[22689]: osdmap e65: 8 total, 8 up, 8 in 2024-04-22T22:24:50.258 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:50 smithi067 ceph-mon[21875]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-04-22T22:24:50.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:50 smithi067 ceph-mon[21875]: osdmap e65: 8 total, 8 up, 8 in 2024-04-22T22:24:50.259 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:50 smithi067 ceph-mon[26633]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-04-22T22:24:50.259 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:50 smithi067 ceph-mon[26633]: osdmap e65: 8 total, 8 up, 8 in 2024-04-22T22:24:50.259 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:50 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager-a[46701]: level=info ts=2024-04-22T22:24:50.151Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.00008857s 2024-04-22T22:24:51.008 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:24:50 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:24:50] "GET /metrics HTTP/1.1" 200 192958 "" "Prometheus/2.33.4" 2024-04-22T22:24:51.109 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:51 smithi086 ceph-mon[22689]: pgmap v56: 65 pgs: 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 639 B/s wr, 4 op/s 2024-04-22T22:24:51.109 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:51 smithi086 ceph-mon[22689]: osdmap e66: 8 total, 8 up, 8 in 2024-04-22T22:24:51.110 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:51 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/2422096363' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-22T22:24:51.110 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:51 smithi086 ceph-mon[22689]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-22T22:24:51.386 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:51 smithi086 systemd[1]: Stopping Ceph prometheus.a for 1ab1a3a8-00f6-11ef-bc93-c7b262605968... 2024-04-22T22:24:51.508 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:51 smithi067 ceph-mon[21875]: pgmap v56: 65 pgs: 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 639 B/s wr, 4 op/s 2024-04-22T22:24:51.508 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:51 smithi067 ceph-mon[21875]: osdmap e66: 8 total, 8 up, 8 in 2024-04-22T22:24:51.508 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:51 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/2422096363' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-22T22:24:51.508 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:51 smithi067 ceph-mon[21875]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-22T22:24:51.509 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:51 smithi067 ceph-mon[26633]: pgmap v56: 65 pgs: 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 639 B/s wr, 4 op/s 2024-04-22T22:24:51.509 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:51 smithi067 ceph-mon[26633]: osdmap e66: 8 total, 8 up, 8 in 2024-04-22T22:24:51.509 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:51 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/2422096363' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-22T22:24:51.509 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:51 smithi067 ceph-mon[26633]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-22T22:24:51.755 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:24:51 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:24:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:24:51.756 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:51 smithi086 bash[39388]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-22T22:24:51.756 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:51 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[37687]: ts=2024-04-22T22:24:51.549Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-22T22:24:51.756 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:51 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[37687]: ts=2024-04-22T22:24:51.549Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-22T22:24:51.756 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:51 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[37687]: ts=2024-04-22T22:24:51.549Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-22T22:24:51.756 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:51 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[37687]: ts=2024-04-22T22:24:51.549Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-22T22:24:51.756 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:51 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[37687]: ts=2024-04-22T22:24:51.549Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-22T22:24:51.756 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:51 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[37687]: ts=2024-04-22T22:24:51.549Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-22T22:24:51.756 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:51 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[37687]: ts=2024-04-22T22:24:51.549Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-22T22:24:51.756 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:51 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[37687]: ts=2024-04-22T22:24:51.549Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-22T22:24:51.756 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:51 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[37687]: ts=2024-04-22T22:24:51.549Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-22T22:24:51.756 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:51 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[37687]: ts=2024-04-22T22:24:51.549Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-22T22:24:51.756 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:51 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[37687]: ts=2024-04-22T22:24:51.549Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-22T22:24:51.757 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:51 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[37687]: ts=2024-04-22T22:24:51.549Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-22T22:24:51.757 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:51 smithi086 podman[39401]: 2024-04-22 22:24:51.559277682 +0000 UTC m=+0.128842564 container died 621e529acde66aee511d27b7a7aea92a9e464b2bb0054a871c1de94cbc2e14a5 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T22:24:52.007 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:51 smithi086 podman[39401]: 2024-04-22 22:24:51.932141184 +0000 UTC m=+0.501706064 container cleanup 621e529acde66aee511d27b7a7aea92a9e464b2bb0054a871c1de94cbc2e14a5 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T22:24:52.007 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:51 smithi086 bash[39401]: ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a 2024-04-22T22:24:52.007 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:51 smithi086 bash[39427]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-22T22:24:52.426 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:52 smithi086 ceph-mon[22689]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-04-22T22:24:52.426 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:52 smithi086 ceph-mon[22689]: osdmap e67: 8 total, 8 up, 8 in 2024-04-22T22:24:52.427 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:52 smithi086 podman[39415]: 2024-04-22 22:24:52.208518124 +0000 UTC m=+0.651975696 container remove 621e529acde66aee511d27b7a7aea92a9e464b2bb0054a871c1de94cbc2e14a5 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T22:24:52.427 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:52 smithi086 systemd[1]: ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-22T22:24:52.427 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:52 smithi086 systemd[1]: Stopped Ceph prometheus.a for 1ab1a3a8-00f6-11ef-bc93-c7b262605968. 2024-04-22T22:24:52.427 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:52 smithi086 systemd[1]: Starting Ceph prometheus.a for 1ab1a3a8-00f6-11ef-bc93-c7b262605968... 2024-04-22T22:24:52.437 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:52 smithi067 ceph-mon[21875]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-04-22T22:24:52.437 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:52 smithi067 ceph-mon[21875]: osdmap e67: 8 total, 8 up, 8 in 2024-04-22T22:24:52.438 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:52 smithi067 ceph-mon[26633]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-04-22T22:24:52.438 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:52 smithi067 ceph-mon[26633]: osdmap e67: 8 total, 8 up, 8 in 2024-04-22T22:24:52.757 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:52 smithi086 podman[39499]: 2024-04-22 22:24:52.340238088 +0000 UTC m=+0.018755603 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-22T22:24:52.757 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:52 smithi086 podman[39499]: 2024-04-22 22:24:52.45094691 +0000 UTC m=+0.129464408 container create 8531ba0d81dc2597a5694b7ee4b7258d37affd4db96bd904c4d07cdc9c949fb0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T22:24:53.325 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:53 smithi086 ceph-mon[22689]: pgmap v59: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 5.7 KiB/s rd, 5 op/s 2024-04-22T22:24:53.325 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:53 smithi086 ceph-mon[22689]: osdmap e68: 8 total, 8 up, 8 in 2024-04-22T22:24:53.325 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:53 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/3033940390' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-22T22:24:53.325 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:53 smithi086 ceph-mon[22689]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-22T22:24:53.508 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:53 smithi067 ceph-mon[26633]: pgmap v59: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 5.7 KiB/s rd, 5 op/s 2024-04-22T22:24:53.509 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:53 smithi067 ceph-mon[26633]: osdmap e68: 8 total, 8 up, 8 in 2024-04-22T22:24:53.509 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:53 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/3033940390' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-22T22:24:53.509 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:53 smithi067 ceph-mon[26633]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-22T22:24:53.509 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:53 smithi067 ceph-mon[21875]: pgmap v59: 97 pgs: 32 unknown, 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 5.7 KiB/s rd, 5 op/s 2024-04-22T22:24:53.509 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:53 smithi067 ceph-mon[21875]: osdmap e68: 8 total, 8 up, 8 in 2024-04-22T22:24:53.509 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:53 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/3033940390' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-22T22:24:53.509 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:53 smithi067 ceph-mon[21875]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-22T22:24:53.600 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:53 smithi086 podman[39499]: 2024-04-22 22:24:53.335334075 +0000 UTC m=+1.013851573 container init 8531ba0d81dc2597a5694b7ee4b7258d37affd4db96bd904c4d07cdc9c949fb0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T22:24:53.600 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:53 smithi086 podman[39499]: 2024-04-22 22:24:53.338637079 +0000 UTC m=+1.017154586 container start 8531ba0d81dc2597a5694b7ee4b7258d37affd4db96bd904c4d07cdc9c949fb0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T22:24:53.600 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:53 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[39513]: ts=2024-04-22T22:24:53.367Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-22T22:24:53.600 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:53 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[39513]: ts=2024-04-22T22:24:53.367Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-22T22:24:53.600 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:53 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[39513]: ts=2024-04-22T22:24:53.367Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-22T22:24:53.600 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:53 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[39513]: ts=2024-04-22T22:24:53.367Z caller=main.go:518 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi086 (none))" 2024-04-22T22:24:53.600 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:53 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[39513]: ts=2024-04-22T22:24:53.367Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-22T22:24:53.600 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:53 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[39513]: ts=2024-04-22T22:24:53.367Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-22T22:24:53.600 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:53 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[39513]: ts=2024-04-22T22:24:53.369Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-22T22:24:53.601 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:53 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[39513]: ts=2024-04-22T22:24:53.369Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-22T22:24:53.601 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:53 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[39513]: ts=2024-04-22T22:24:53.371Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-22T22:24:53.601 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:53 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[39513]: ts=2024-04-22T22:24:53.373Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-22T22:24:53.601 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:53 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[39513]: ts=2024-04-22T22:24:53.373Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.834µs 2024-04-22T22:24:53.601 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:53 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[39513]: ts=2024-04-22T22:24:53.373Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-22T22:24:54.007 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:53 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[39513]: ts=2024-04-22T22:24:53.609Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-04-22T22:24:54.007 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:53 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[39513]: ts=2024-04-22T22:24:53.609Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-04-22T22:24:54.007 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:53 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[39513]: ts=2024-04-22T22:24:53.609Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=28.054µs wal_replay_duration=236.236391ms total_replay_duration=236.277127ms 2024-04-22T22:24:54.008 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:53 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[39513]: ts=2024-04-22T22:24:53.612Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-22T22:24:54.008 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:53 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[39513]: ts=2024-04-22T22:24:53.612Z caller=main.go:947 level=info msg="TSDB started" 2024-04-22T22:24:54.008 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:53 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[39513]: ts=2024-04-22T22:24:53.612Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-22T22:24:54.008 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:53 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[39513]: ts=2024-04-22T22:24:53.628Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=16.129525ms db_storage=973ns remote_storage=1.439µs web_handler=386ns query_engine=739ns scrape=400.934µs scrape_sd=27.905µs notify=22.919µs notify_sd=6.813µs rules=15.418368ms 2024-04-22T22:24:54.008 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:53 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[39513]: ts=2024-04-22T22:24:53.628Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-22T22:24:54.311 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:54 smithi086 ceph-mon[22689]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-04-22T22:24:54.311 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:54 smithi086 ceph-mon[22689]: osdmap e69: 8 total, 8 up, 8 in 2024-04-22T22:24:54.311 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:54 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/3033940390' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-22T22:24:54.311 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:54 smithi086 ceph-mon[22689]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-22T22:24:54.312 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:54 smithi086 bash[39499]: 8531ba0d81dc2597a5694b7ee4b7258d37affd4db96bd904c4d07cdc9c949fb0 2024-04-22T22:24:54.312 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:24:54 smithi086 systemd[1]: Started Ceph prometheus.a for 1ab1a3a8-00f6-11ef-bc93-c7b262605968. 2024-04-22T22:24:54.312 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:54 smithi067 ceph-mon[21875]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-04-22T22:24:54.312 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:54 smithi067 ceph-mon[21875]: osdmap e69: 8 total, 8 up, 8 in 2024-04-22T22:24:54.312 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:54 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/3033940390' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-22T22:24:54.312 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:54 smithi067 ceph-mon[21875]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-22T22:24:54.312 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:54 smithi067 ceph-mon[26633]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-04-22T22:24:54.313 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:54 smithi067 ceph-mon[26633]: osdmap e69: 8 total, 8 up, 8 in 2024-04-22T22:24:54.313 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:54 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/3033940390' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-22T22:24:54.313 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:54 smithi067 ceph-mon[26633]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-22T22:24:55.090 INFO:teuthology.orchestra.run.smithi067.stdout:{ 2024-04-22T22:24:55.090 INFO:teuthology.orchestra.run.smithi067.stdout: "id": "548f51cd-d0c3-44a6-9df8-d1fc14271de3", 2024-04-22T22:24:55.090 INFO:teuthology.orchestra.run.smithi067.stdout: "epoch": 1, 2024-04-22T22:24:55.091 INFO:teuthology.orchestra.run.smithi067.stdout: "predecessor_uuid": "a73e1e36-577c-423f-8623-0b51247ebf7f", 2024-04-22T22:24:55.091 INFO:teuthology.orchestra.run.smithi067.stdout: "sync_status": [], 2024-04-22T22:24:55.091 INFO:teuthology.orchestra.run.smithi067.stdout: "period_map": { 2024-04-22T22:24:55.091 INFO:teuthology.orchestra.run.smithi067.stdout: "id": "548f51cd-d0c3-44a6-9df8-d1fc14271de3", 2024-04-22T22:24:55.091 INFO:teuthology.orchestra.run.smithi067.stdout: "zonegroups": [ 2024-04-22T22:24:55.091 INFO:teuthology.orchestra.run.smithi067.stdout: { 2024-04-22T22:24:55.091 INFO:teuthology.orchestra.run.smithi067.stdout: "id": "115d9175-dd64-4999-913d-15472a1de9cd", 2024-04-22T22:24:55.091 INFO:teuthology.orchestra.run.smithi067.stdout: "name": "default", 2024-04-22T22:24:55.091 INFO:teuthology.orchestra.run.smithi067.stdout: "api_name": "default", 2024-04-22T22:24:55.091 INFO:teuthology.orchestra.run.smithi067.stdout: "is_master": "true", 2024-04-22T22:24:55.091 INFO:teuthology.orchestra.run.smithi067.stdout: "endpoints": [], 2024-04-22T22:24:55.091 INFO:teuthology.orchestra.run.smithi067.stdout: "hostnames": [], 2024-04-22T22:24:55.092 INFO:teuthology.orchestra.run.smithi067.stdout: "hostnames_s3website": [], 2024-04-22T22:24:55.092 INFO:teuthology.orchestra.run.smithi067.stdout: "master_zone": "8a3bce13-588b-405a-b6eb-772a44cd8dfb", 2024-04-22T22:24:55.092 INFO:teuthology.orchestra.run.smithi067.stdout: "zones": [ 2024-04-22T22:24:55.092 INFO:teuthology.orchestra.run.smithi067.stdout: { 2024-04-22T22:24:55.092 INFO:teuthology.orchestra.run.smithi067.stdout: "id": "8a3bce13-588b-405a-b6eb-772a44cd8dfb", 2024-04-22T22:24:55.092 INFO:teuthology.orchestra.run.smithi067.stdout: "name": "z", 2024-04-22T22:24:55.092 INFO:teuthology.orchestra.run.smithi067.stdout: "endpoints": [], 2024-04-22T22:24:55.092 INFO:teuthology.orchestra.run.smithi067.stdout: "log_meta": "false", 2024-04-22T22:24:55.092 INFO:teuthology.orchestra.run.smithi067.stdout: "log_data": "false", 2024-04-22T22:24:55.092 INFO:teuthology.orchestra.run.smithi067.stdout: "bucket_index_max_shards": 11, 2024-04-22T22:24:55.092 INFO:teuthology.orchestra.run.smithi067.stdout: "read_only": "false", 2024-04-22T22:24:55.092 INFO:teuthology.orchestra.run.smithi067.stdout: "tier_type": "", 2024-04-22T22:24:55.092 INFO:teuthology.orchestra.run.smithi067.stdout: "sync_from_all": "true", 2024-04-22T22:24:55.092 INFO:teuthology.orchestra.run.smithi067.stdout: "sync_from": [], 2024-04-22T22:24:55.093 INFO:teuthology.orchestra.run.smithi067.stdout: "redirect_zone": "" 2024-04-22T22:24:55.093 INFO:teuthology.orchestra.run.smithi067.stdout: } 2024-04-22T22:24:55.093 INFO:teuthology.orchestra.run.smithi067.stdout: ], 2024-04-22T22:24:55.093 INFO:teuthology.orchestra.run.smithi067.stdout: "placement_targets": [ 2024-04-22T22:24:55.093 INFO:teuthology.orchestra.run.smithi067.stdout: { 2024-04-22T22:24:55.093 INFO:teuthology.orchestra.run.smithi067.stdout: "name": "default-placement", 2024-04-22T22:24:55.093 INFO:teuthology.orchestra.run.smithi067.stdout: "tags": [], 2024-04-22T22:24:55.093 INFO:teuthology.orchestra.run.smithi067.stdout: "storage_classes": [ 2024-04-22T22:24:55.093 INFO:teuthology.orchestra.run.smithi067.stdout: "STANDARD" 2024-04-22T22:24:55.093 INFO:teuthology.orchestra.run.smithi067.stdout: ] 2024-04-22T22:24:55.093 INFO:teuthology.orchestra.run.smithi067.stdout: } 2024-04-22T22:24:55.093 INFO:teuthology.orchestra.run.smithi067.stdout: ], 2024-04-22T22:24:55.093 INFO:teuthology.orchestra.run.smithi067.stdout: "default_placement": "default-placement", 2024-04-22T22:24:55.094 INFO:teuthology.orchestra.run.smithi067.stdout: "realm_id": "5415be30-74e7-47c2-9bf7-2ba8315931bf", 2024-04-22T22:24:55.094 INFO:teuthology.orchestra.run.smithi067.stdout: "sync_policy": { 2024-04-22T22:24:55.094 INFO:teuthology.orchestra.run.smithi067.stdout: "groups": [] 2024-04-22T22:24:55.094 INFO:teuthology.orchestra.run.smithi067.stdout: } 2024-04-22T22:24:55.094 INFO:teuthology.orchestra.run.smithi067.stdout: } 2024-04-22T22:24:55.094 INFO:teuthology.orchestra.run.smithi067.stdout: ], 2024-04-22T22:24:55.094 INFO:teuthology.orchestra.run.smithi067.stdout: "short_zone_ids": [ 2024-04-22T22:24:55.094 INFO:teuthology.orchestra.run.smithi067.stdout: { 2024-04-22T22:24:55.094 INFO:teuthology.orchestra.run.smithi067.stdout: "key": "8a3bce13-588b-405a-b6eb-772a44cd8dfb", 2024-04-22T22:24:55.094 INFO:teuthology.orchestra.run.smithi067.stdout: "val": 2575193685 2024-04-22T22:24:55.095 INFO:teuthology.orchestra.run.smithi067.stdout: } 2024-04-22T22:24:55.095 INFO:teuthology.orchestra.run.smithi067.stdout: ] 2024-04-22T22:24:55.095 INFO:teuthology.orchestra.run.smithi067.stdout: }, 2024-04-22T22:24:55.095 INFO:teuthology.orchestra.run.smithi067.stdout: "master_zonegroup": "115d9175-dd64-4999-913d-15472a1de9cd", 2024-04-22T22:24:55.095 INFO:teuthology.orchestra.run.smithi067.stdout: "master_zone": "8a3bce13-588b-405a-b6eb-772a44cd8dfb", 2024-04-22T22:24:55.095 INFO:teuthology.orchestra.run.smithi067.stdout: "period_config": { 2024-04-22T22:24:55.095 INFO:teuthology.orchestra.run.smithi067.stdout: "bucket_quota": { 2024-04-22T22:24:55.095 INFO:teuthology.orchestra.run.smithi067.stdout: "enabled": false, 2024-04-22T22:24:55.095 INFO:teuthology.orchestra.run.smithi067.stdout: "check_on_raw": false, 2024-04-22T22:24:55.095 INFO:teuthology.orchestra.run.smithi067.stdout: "max_size": -1, 2024-04-22T22:24:55.095 INFO:teuthology.orchestra.run.smithi067.stdout: "max_size_kb": 0, 2024-04-22T22:24:55.096 INFO:teuthology.orchestra.run.smithi067.stdout: "max_objects": -1 2024-04-22T22:24:55.096 INFO:teuthology.orchestra.run.smithi067.stdout: }, 2024-04-22T22:24:55.096 INFO:teuthology.orchestra.run.smithi067.stdout: "user_quota": { 2024-04-22T22:24:55.096 INFO:teuthology.orchestra.run.smithi067.stdout: "enabled": false, 2024-04-22T22:24:55.096 INFO:teuthology.orchestra.run.smithi067.stdout: "check_on_raw": false, 2024-04-22T22:24:55.096 INFO:teuthology.orchestra.run.smithi067.stdout: "max_size": -1, 2024-04-22T22:24:55.096 INFO:teuthology.orchestra.run.smithi067.stdout: "max_size_kb": 0, 2024-04-22T22:24:55.096 INFO:teuthology.orchestra.run.smithi067.stdout: "max_objects": -1 2024-04-22T22:24:55.096 INFO:teuthology.orchestra.run.smithi067.stdout: }, 2024-04-22T22:24:55.096 INFO:teuthology.orchestra.run.smithi067.stdout: "user_ratelimit": { 2024-04-22T22:24:55.096 INFO:teuthology.orchestra.run.smithi067.stdout: "max_read_ops": 0, 2024-04-22T22:24:55.096 INFO:teuthology.orchestra.run.smithi067.stdout: "max_write_ops": 0, 2024-04-22T22:24:55.096 INFO:teuthology.orchestra.run.smithi067.stdout: "max_read_bytes": 0, 2024-04-22T22:24:55.096 INFO:teuthology.orchestra.run.smithi067.stdout: "max_write_bytes": 0, 2024-04-22T22:24:55.097 INFO:teuthology.orchestra.run.smithi067.stdout: "enabled": false 2024-04-22T22:24:55.097 INFO:teuthology.orchestra.run.smithi067.stdout: }, 2024-04-22T22:24:55.097 INFO:teuthology.orchestra.run.smithi067.stdout: "bucket_ratelimit": { 2024-04-22T22:24:55.097 INFO:teuthology.orchestra.run.smithi067.stdout: "max_read_ops": 0, 2024-04-22T22:24:55.097 INFO:teuthology.orchestra.run.smithi067.stdout: "max_write_ops": 0, 2024-04-22T22:24:55.097 INFO:teuthology.orchestra.run.smithi067.stdout: "max_read_bytes": 0, 2024-04-22T22:24:55.097 INFO:teuthology.orchestra.run.smithi067.stdout: "max_write_bytes": 0, 2024-04-22T22:24:55.097 INFO:teuthology.orchestra.run.smithi067.stdout: "enabled": false 2024-04-22T22:24:55.097 INFO:teuthology.orchestra.run.smithi067.stdout: }, 2024-04-22T22:24:55.097 INFO:teuthology.orchestra.run.smithi067.stdout: "anonymous_ratelimit": { 2024-04-22T22:24:55.097 INFO:teuthology.orchestra.run.smithi067.stdout: "max_read_ops": 0, 2024-04-22T22:24:55.097 INFO:teuthology.orchestra.run.smithi067.stdout: "max_write_ops": 0, 2024-04-22T22:24:55.097 INFO:teuthology.orchestra.run.smithi067.stdout: "max_read_bytes": 0, 2024-04-22T22:24:55.098 INFO:teuthology.orchestra.run.smithi067.stdout: "max_write_bytes": 0, 2024-04-22T22:24:55.098 INFO:teuthology.orchestra.run.smithi067.stdout: "enabled": false 2024-04-22T22:24:55.098 INFO:teuthology.orchestra.run.smithi067.stdout: } 2024-04-22T22:24:55.098 INFO:teuthology.orchestra.run.smithi067.stdout: }, 2024-04-22T22:24:55.098 INFO:teuthology.orchestra.run.smithi067.stdout: "realm_id": "5415be30-74e7-47c2-9bf7-2ba8315931bf", 2024-04-22T22:24:55.098 INFO:teuthology.orchestra.run.smithi067.stdout: "realm_name": "r", 2024-04-22T22:24:55.098 INFO:teuthology.orchestra.run.smithi067.stdout: "realm_epoch": 2 2024-04-22T22:24:55.098 INFO:teuthology.orchestra.run.smithi067.stdout:} 2024-04-22T22:24:55.099 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[21875]: pgmap v62: 129 pgs: 64 unknown, 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:55.099 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[21875]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-04-22T22:24:55.099 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[21875]: osdmap e70: 8 total, 8 up, 8 in 2024-04-22T22:24:55.099 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/3033940390' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-22T22:24:55.099 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[21875]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-22T22:24:55.099 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:55.099 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T22:24:55.099 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[21875]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T22:24:55.099 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.67:9093"}]: dispatch 2024-04-22T22:24:55.099 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[21875]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.67:9093"}]: dispatch 2024-04-22T22:24:55.099 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:55.100 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T22:24:55.100 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[21875]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T22:24:55.100 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.86:9095"}]: dispatch 2024-04-22T22:24:55.100 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[21875]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.86:9095"}]: dispatch 2024-04-22T22:24:55.100 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:55.100 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T22:24:55.100 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[21875]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T22:24:55.100 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.86:3000"}]: dispatch 2024-04-22T22:24:55.100 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[21875]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.86:3000"}]: dispatch 2024-04-22T22:24:55.100 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:55.100 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:24:55.100 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:24:55.101 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[26633]: pgmap v62: 129 pgs: 64 unknown, 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:55.101 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[26633]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-04-22T22:24:55.101 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[26633]: osdmap e70: 8 total, 8 up, 8 in 2024-04-22T22:24:55.101 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/3033940390' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-22T22:24:55.101 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[26633]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-22T22:24:55.101 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:55.101 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T22:24:55.101 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[26633]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T22:24:55.101 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.67:9093"}]: dispatch 2024-04-22T22:24:55.101 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[26633]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.67:9093"}]: dispatch 2024-04-22T22:24:55.101 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:55.102 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T22:24:55.102 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[26633]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T22:24:55.102 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.86:9095"}]: dispatch 2024-04-22T22:24:55.102 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[26633]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.86:9095"}]: dispatch 2024-04-22T22:24:55.102 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:55.102 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T22:24:55.102 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[26633]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T22:24:55.102 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.86:3000"}]: dispatch 2024-04-22T22:24:55.102 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[26633]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.86:3000"}]: dispatch 2024-04-22T22:24:55.102 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:55.102 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:24:55.102 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:55 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:24:55.508 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:55 smithi086 ceph-mon[22689]: pgmap v62: 129 pgs: 64 unknown, 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:24:55.508 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:55 smithi086 ceph-mon[22689]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-04-22T22:24:55.508 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:55 smithi086 ceph-mon[22689]: osdmap e70: 8 total, 8 up, 8 in 2024-04-22T22:24:55.508 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:55 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/3033940390' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-22T22:24:55.508 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:55 smithi086 ceph-mon[22689]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-22T22:24:55.508 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:55 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:55.508 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:55 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T22:24:55.508 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:55 smithi086 ceph-mon[22689]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-22T22:24:55.508 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:55 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.67:9093"}]: dispatch 2024-04-22T22:24:55.508 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:55 smithi086 ceph-mon[22689]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.67:9093"}]: dispatch 2024-04-22T22:24:55.508 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:55 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:55.508 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:55 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T22:24:55.508 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:55 smithi086 ceph-mon[22689]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-22T22:24:55.508 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:55 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.86:9095"}]: dispatch 2024-04-22T22:24:55.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:55 smithi086 ceph-mon[22689]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.86:9095"}]: dispatch 2024-04-22T22:24:55.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:55 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:55.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:55 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T22:24:55.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:55 smithi086 ceph-mon[22689]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-22T22:24:55.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:55 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.86:3000"}]: dispatch 2024-04-22T22:24:55.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:55 smithi086 ceph-mon[22689]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.86:3000"}]: dispatch 2024-04-22T22:24:55.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:55 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:55.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:55 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:24:55.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:55 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:24:55.938 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -e sha1=430e09df97c8fc7dc2b2ae424f68ed11366c540f -- bash -c 'ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000' 2024-04-22T22:24:56.258 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:56 smithi067 ceph-mon[21875]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-04-22T22:24:56.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:56 smithi067 ceph-mon[21875]: osdmap e71: 8 total, 8 up, 8 in 2024-04-22T22:24:56.259 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:56 smithi067 ceph-mon[26633]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-04-22T22:24:56.259 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:56 smithi067 ceph-mon[26633]: osdmap e71: 8 total, 8 up, 8 in 2024-04-22T22:24:56.395 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:56 smithi086 ceph-mon[22689]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-04-22T22:24:56.395 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:56 smithi086 ceph-mon[22689]: osdmap e71: 8 total, 8 up, 8 in 2024-04-22T22:24:57.394 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:57 smithi067 ceph-mon[21875]: pgmap v65: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-04-22T22:24:57.395 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:57 smithi067 ceph-mon[26633]: pgmap v65: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-04-22T22:24:57.487 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:57 smithi086 ceph-mon[22689]: pgmap v65: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-04-22T22:24:57.980 INFO:teuthology.orchestra.run.smithi067.stdout:Scheduled rgw.foo update... 2024-04-22T22:24:58.259 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:24:58 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager-a[46701]: level=info ts=2024-04-22T22:24:58.153Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.00138704s 2024-04-22T22:24:59.218 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:58 smithi067 ceph-mon[21875]: pgmap v66: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 192 B/s rd, 384 B/s wr, 1 op/s 2024-04-22T22:24:59.218 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:58 smithi067 ceph-mon[21875]: from='client.24598 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:24:59.218 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:58 smithi067 ceph-mon[21875]: Saving service rgw.foo spec with placement count:2 2024-04-22T22:24:59.218 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:24:58 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:59.218 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:58 smithi067 ceph-mon[26633]: pgmap v66: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 192 B/s rd, 384 B/s wr, 1 op/s 2024-04-22T22:24:59.218 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:58 smithi067 ceph-mon[26633]: from='client.24598 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:24:59.219 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:58 smithi067 ceph-mon[26633]: Saving service rgw.foo spec with placement count:2 2024-04-22T22:24:59.219 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:24:58 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:24:59.243 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -e sha1=430e09df97c8fc7dc2b2ae424f68ed11366c540f -- bash -c 'ceph osd pool create foo' 2024-04-22T22:24:59.257 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:58 smithi086 ceph-mon[22689]: pgmap v66: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 192 B/s rd, 384 B/s wr, 1 op/s 2024-04-22T22:24:59.257 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:58 smithi086 ceph-mon[22689]: from='client.24598 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:24:59.257 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:58 smithi086 ceph-mon[22689]: Saving service rgw.foo spec with placement count:2 2024-04-22T22:24:59.257 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:24:58 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:00.758 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:25:00 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:25:00] "GET /metrics HTTP/1.1" 200 196804 "" "Prometheus/2.33.4" 2024-04-22T22:25:01.121 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:00 smithi067 ceph-mon[21875]: pgmap v67: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.4 KiB/s wr, 3 op/s 2024-04-22T22:25:01.121 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:00 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:01.121 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:00 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:01.121 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:00 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:01.121 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:00 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:01.121 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:00 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:01.121 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:00 smithi067 ceph-mon[21875]: Saving service rgw.foo spec with placement count:2 2024-04-22T22:25:01.121 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:00 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:01.122 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:00 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi067.bekyzf", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T22:25:01.122 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:00 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi067.bekyzf", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-22T22:25:01.122 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:00 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:01.122 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:00 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:25:01.122 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:00 smithi067 ceph-mon[21875]: Deploying daemon rgw.foo.smithi067.bekyzf on smithi067 2024-04-22T22:25:01.122 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:00 smithi067 ceph-mon[26633]: pgmap v67: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.4 KiB/s wr, 3 op/s 2024-04-22T22:25:01.122 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:00 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:01.122 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:00 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:01.122 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:00 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:01.122 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:00 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:01.122 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:00 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:01.122 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:00 smithi067 ceph-mon[26633]: Saving service rgw.foo spec with placement count:2 2024-04-22T22:25:01.122 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:00 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:01.123 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:00 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi067.bekyzf", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T22:25:01.123 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:00 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi067.bekyzf", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-22T22:25:01.123 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:00 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:01.123 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:00 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:25:01.123 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:00 smithi067 ceph-mon[26633]: Deploying daemon rgw.foo.smithi067.bekyzf on smithi067 2024-04-22T22:25:01.257 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:00 smithi086 ceph-mon[22689]: pgmap v67: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.4 KiB/s wr, 3 op/s 2024-04-22T22:25:01.265 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:00 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:01.265 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:00 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:01.265 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:00 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:01.265 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:00 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:01.265 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:00 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:01.265 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:00 smithi086 ceph-mon[22689]: Saving service rgw.foo spec with placement count:2 2024-04-22T22:25:01.265 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:00 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:01.265 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:00 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi067.bekyzf", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T22:25:01.265 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:00 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi067.bekyzf", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-22T22:25:01.265 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:00 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:01.265 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:00 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:25:01.265 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:00 smithi086 ceph-mon[22689]: Deploying daemon rgw.foo.smithi067.bekyzf on smithi067 2024-04-22T22:25:01.757 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:25:01 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:25:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:25:02.271 INFO:teuthology.orchestra.run.smithi067.stderr:pool 'foo' created 2024-04-22T22:25:02.509 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:02 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/4197371914' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-22T22:25:02.509 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:02 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/4197371914' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-22T22:25:02.757 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:02 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/4197371914' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-22T22:25:02.974 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -e sha1=430e09df97c8fc7dc2b2ae424f68ed11366c540f -- bash -c 'rbd pool init foo' 2024-04-22T22:25:03.535 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:03 smithi067 ceph-mon[21875]: pgmap v68: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1.1 KiB/s wr, 2 op/s 2024-04-22T22:25:03.536 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:03 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/4197371914' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-04-22T22:25:03.536 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:03 smithi067 ceph-mon[21875]: osdmap e72: 8 total, 8 up, 8 in 2024-04-22T22:25:03.536 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:03 smithi067 ceph-mon[21875]: osdmap e73: 8 total, 8 up, 8 in 2024-04-22T22:25:03.536 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:03 smithi067 ceph-mon[26633]: pgmap v68: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1.1 KiB/s wr, 2 op/s 2024-04-22T22:25:03.536 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:03 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/4197371914' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-04-22T22:25:03.537 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:03 smithi067 ceph-mon[26633]: osdmap e72: 8 total, 8 up, 8 in 2024-04-22T22:25:03.537 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:03 smithi067 ceph-mon[26633]: osdmap e73: 8 total, 8 up, 8 in 2024-04-22T22:25:03.757 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:03 smithi086 ceph-mon[22689]: pgmap v68: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1.1 KiB/s wr, 2 op/s 2024-04-22T22:25:03.757 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:03 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/4197371914' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-04-22T22:25:03.757 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:03 smithi086 ceph-mon[22689]: osdmap e72: 8 total, 8 up, 8 in 2024-04-22T22:25:03.757 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:03 smithi086 ceph-mon[22689]: osdmap e73: 8 total, 8 up, 8 in 2024-04-22T22:25:05.509 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:05 smithi067 ceph-mon[26633]: pgmap v71: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 895 B/s wr, 2 op/s 2024-04-22T22:25:05.509 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:05 smithi067 ceph-mon[26633]: osdmap e74: 8 total, 8 up, 8 in 2024-04-22T22:25:05.509 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:05 smithi067 ceph-mon[21875]: pgmap v71: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 895 B/s wr, 2 op/s 2024-04-22T22:25:05.509 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:05 smithi067 ceph-mon[21875]: osdmap e74: 8 total, 8 up, 8 in 2024-04-22T22:25:05.757 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:05 smithi086 ceph-mon[22689]: pgmap v71: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 895 B/s rd, 895 B/s wr, 2 op/s 2024-04-22T22:25:05.757 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:05 smithi086 ceph-mon[22689]: osdmap e74: 8 total, 8 up, 8 in 2024-04-22T22:25:06.499 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:06 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/2536580726' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-22T22:25:06.500 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:06 smithi086 ceph-mon[22689]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-22T22:25:06.500 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:06 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:06.500 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:06 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi086.cofxku", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T22:25:06.500 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:06 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi086.cofxku", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-22T22:25:06.500 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:06 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:06.500 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:06 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:25:06.759 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:06 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/2536580726' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-22T22:25:06.759 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:06 smithi067 ceph-mon[21875]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-22T22:25:06.759 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:06 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:06.759 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:06 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi086.cofxku", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T22:25:06.759 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:06 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi086.cofxku", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-22T22:25:06.759 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:06 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:06.759 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:06 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:25:06.759 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:06 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/2536580726' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-22T22:25:06.759 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:06 smithi067 ceph-mon[26633]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-22T22:25:06.759 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:06 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:06.759 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:06 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi086.cofxku", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-22T22:25:06.759 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:06 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi086.cofxku", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-22T22:25:06.760 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:06 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:06.760 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:06 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:25:07.611 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:07 smithi086 ceph-mon[22689]: pgmap v73: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:25:07.611 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:07 smithi086 ceph-mon[22689]: Deploying daemon rgw.foo.smithi086.cofxku on smithi086 2024-04-22T22:25:07.611 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:07 smithi086 ceph-mon[22689]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-04-22T22:25:07.611 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:07 smithi086 ceph-mon[22689]: osdmap e75: 8 total, 8 up, 8 in 2024-04-22T22:25:07.759 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:07 smithi067 ceph-mon[21875]: pgmap v73: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:25:07.759 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:07 smithi067 ceph-mon[21875]: Deploying daemon rgw.foo.smithi086.cofxku on smithi086 2024-04-22T22:25:07.759 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:07 smithi067 ceph-mon[21875]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-04-22T22:25:07.759 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:07 smithi067 ceph-mon[21875]: osdmap e75: 8 total, 8 up, 8 in 2024-04-22T22:25:07.759 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:07 smithi067 ceph-mon[26633]: pgmap v73: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:25:07.759 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:07 smithi067 ceph-mon[26633]: Deploying daemon rgw.foo.smithi086.cofxku on smithi086 2024-04-22T22:25:07.759 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:07 smithi067 ceph-mon[26633]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-04-22T22:25:07.759 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:07 smithi067 ceph-mon[26633]: osdmap e75: 8 total, 8 up, 8 in 2024-04-22T22:25:08.468 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:08 smithi067 ceph-mon[21875]: osdmap e76: 8 total, 8 up, 8 in 2024-04-22T22:25:08.468 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:08 smithi067 ceph-mon[26633]: osdmap e76: 8 total, 8 up, 8 in 2024-04-22T22:25:08.757 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:08 smithi086 ceph-mon[22689]: osdmap e76: 8 total, 8 up, 8 in 2024-04-22T22:25:09.050 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -e sha1=430e09df97c8fc7dc2b2ae424f68ed11366c540f -- bash -c 'ceph orch apply iscsi foo u p' 2024-04-22T22:25:09.507 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:09 smithi086 ceph-mon[22689]: pgmap v76: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:25:09.508 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:09 smithi086 ceph-mon[22689]: osdmap e77: 8 total, 8 up, 8 in 2024-04-22T22:25:09.508 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:09 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:09.508 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:09 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:25:09.508 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:09 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:25:09.590 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:09 smithi067 ceph-mon[21875]: pgmap v76: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:25:09.591 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:09 smithi067 ceph-mon[21875]: osdmap e77: 8 total, 8 up, 8 in 2024-04-22T22:25:09.591 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:09 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:09.591 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:09 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:25:09.591 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:09 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:25:09.591 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:09 smithi067 ceph-mon[26633]: pgmap v76: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:25:09.591 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:09 smithi067 ceph-mon[26633]: osdmap e77: 8 total, 8 up, 8 in 2024-04-22T22:25:09.591 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:09 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:09.591 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:09 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:25:09.591 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:09 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:25:10.759 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:25:10 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:25:10] "GET /metrics HTTP/1.1" 200 196804 "" "Prometheus/2.33.4" 2024-04-22T22:25:11.607 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:11 smithi086 ceph-mon[22689]: pgmap v78: 161 pgs: 161 active+clean; 584 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 219 KiB/s rd, 7.1 KiB/s wr, 418 op/s 2024-04-22T22:25:11.608 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:25:11 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:25:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:25:11.759 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:11 smithi067 ceph-mon[21875]: pgmap v78: 161 pgs: 161 active+clean; 584 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 219 KiB/s rd, 7.1 KiB/s wr, 418 op/s 2024-04-22T22:25:11.759 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:11 smithi067 ceph-mon[26633]: pgmap v78: 161 pgs: 161 active+clean; 584 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 219 KiB/s rd, 7.1 KiB/s wr, 418 op/s 2024-04-22T22:25:11.775 INFO:teuthology.orchestra.run.smithi067.stdout:Scheduled iscsi.foo update... 2024-04-22T22:25:12.960 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:12 smithi067 ceph-mon[21875]: pgmap v79: 161 pgs: 161 active+clean; 584 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 185 KiB/s rd, 6.0 KiB/s wr, 352 op/s 2024-04-22T22:25:12.961 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:12 smithi067 ceph-mon[21875]: from='client.24658 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:25:12.961 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:12 smithi067 ceph-mon[21875]: Saving service iscsi.foo spec with placement count:1 2024-04-22T22:25:12.961 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:12 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:12.961 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:12 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:12.961 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:12 smithi067 ceph-mon[26633]: pgmap v79: 161 pgs: 161 active+clean; 584 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 185 KiB/s rd, 6.0 KiB/s wr, 352 op/s 2024-04-22T22:25:12.961 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:12 smithi067 ceph-mon[26633]: from='client.24658 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:25:12.961 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:12 smithi067 ceph-mon[26633]: Saving service iscsi.foo spec with placement count:1 2024-04-22T22:25:12.961 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:12 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:12.961 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:12 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:12.989 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -e sha1=430e09df97c8fc7dc2b2ae424f68ed11366c540f -- bash -c 'sleep 180' 2024-04-22T22:25:13.257 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:12 smithi086 ceph-mon[22689]: pgmap v79: 161 pgs: 161 active+clean; 584 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 185 KiB/s rd, 6.0 KiB/s wr, 352 op/s 2024-04-22T22:25:13.257 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:12 smithi086 ceph-mon[22689]: from='client.24658 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:25:13.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:12 smithi086 ceph-mon[22689]: Saving service iscsi.foo spec with placement count:1 2024-04-22T22:25:13.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:12 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:13.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:12 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:14.009 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:13 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]: dispatch 2024-04-22T22:25:14.009 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:13 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2024-04-22T22:25:14.009 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:13 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-04-22T22:25:14.009 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:13 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2024-04-22T22:25:14.009 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:13 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:25:14.009 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:13 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:25:14.009 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:13 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:14.009 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:13 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:14.010 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:13 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]: dispatch 2024-04-22T22:25:14.010 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:13 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2024-04-22T22:25:14.010 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:13 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-04-22T22:25:14.010 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:13 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2024-04-22T22:25:14.010 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:13 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:25:14.010 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:13 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:25:14.010 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:13 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:14.010 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:13 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:14.257 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:13 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]: dispatch 2024-04-22T22:25:14.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:13 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2024-04-22T22:25:14.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:13 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-04-22T22:25:14.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:13 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2024-04-22T22:25:14.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:13 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:25:14.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:13 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:25:14.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:13 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:14.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:13 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:15.052 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:14 smithi067 ceph-mon[21875]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 157 KiB/s rd, 5.1 KiB/s wr, 299 op/s 2024-04-22T22:25:15.052 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:14 smithi067 ceph-mon[21875]: Checking dashboard <-> RGW credentials 2024-04-22T22:25:15.052 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:14 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]': finished 2024-04-22T22:25:15.052 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:14 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2024-04-22T22:25:15.052 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:14 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-04-22T22:25:15.052 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:14 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2024-04-22T22:25:15.052 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:14 smithi067 ceph-mon[21875]: osdmap e78: 8 total, 8 up, 8 in 2024-04-22T22:25:15.053 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:14 smithi067 ceph-mon[21875]: osdmap e79: 8 total, 8 up, 8 in 2024-04-22T22:25:15.053 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:14 smithi067 ceph-mon[26633]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 157 KiB/s rd, 5.1 KiB/s wr, 299 op/s 2024-04-22T22:25:15.053 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:14 smithi067 ceph-mon[26633]: Checking dashboard <-> RGW credentials 2024-04-22T22:25:15.053 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:14 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]': finished 2024-04-22T22:25:15.053 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:14 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2024-04-22T22:25:15.053 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:14 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-04-22T22:25:15.053 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:14 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2024-04-22T22:25:15.053 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:14 smithi067 ceph-mon[26633]: osdmap e78: 8 total, 8 up, 8 in 2024-04-22T22:25:15.053 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:14 smithi067 ceph-mon[26633]: osdmap e79: 8 total, 8 up, 8 in 2024-04-22T22:25:15.257 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:14 smithi086 ceph-mon[22689]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 157 KiB/s rd, 5.1 KiB/s wr, 299 op/s 2024-04-22T22:25:15.257 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:14 smithi086 ceph-mon[22689]: Checking dashboard <-> RGW credentials 2024-04-22T22:25:15.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:14 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.12", "id": [7, 2]}]': finished 2024-04-22T22:25:15.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:14 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2024-04-22T22:25:15.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:14 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-04-22T22:25:15.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:14 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2024-04-22T22:25:15.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:14 smithi086 ceph-mon[22689]: osdmap e78: 8 total, 8 up, 8 in 2024-04-22T22:25:15.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:14 smithi086 ceph-mon[22689]: osdmap e79: 8 total, 8 up, 8 in 2024-04-22T22:25:16.758 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:16 smithi086 ceph-mon[22689]: pgmap v83: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 157 KiB/s rd, 5.1 KiB/s wr, 300 op/s 2024-04-22T22:25:16.758 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:16 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:16.758 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:16 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:16.758 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:16 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:16.758 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:16 smithi086 ceph-mon[22689]: Checking pool "foo" exists for service iscsi.foo 2024-04-22T22:25:16.758 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:16 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi067.rrokcu", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T22:25:16.758 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:16 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi067.rrokcu", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-04-22T22:25:16.758 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:16 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:25:16.758 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:16 smithi086 ceph-mon[22689]: Deploying daemon iscsi.foo.smithi067.rrokcu on smithi067 2024-04-22T22:25:16.759 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:16 smithi067 ceph-mon[26633]: pgmap v83: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 157 KiB/s rd, 5.1 KiB/s wr, 300 op/s 2024-04-22T22:25:16.759 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:16 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:16.759 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:16 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:16.759 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:16 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:16.759 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:16 smithi067 ceph-mon[26633]: Checking pool "foo" exists for service iscsi.foo 2024-04-22T22:25:16.759 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:16 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi067.rrokcu", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T22:25:16.760 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:16 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi067.rrokcu", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-04-22T22:25:16.760 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:16 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:25:16.760 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:16 smithi067 ceph-mon[26633]: Deploying daemon iscsi.foo.smithi067.rrokcu on smithi067 2024-04-22T22:25:16.760 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:16 smithi067 ceph-mon[21875]: pgmap v83: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 157 KiB/s rd, 5.1 KiB/s wr, 300 op/s 2024-04-22T22:25:16.760 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:16 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:16.760 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:16 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:16.760 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:16 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:16.760 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:16 smithi067 ceph-mon[21875]: Checking pool "foo" exists for service iscsi.foo 2024-04-22T22:25:16.760 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:16 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi067.rrokcu", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-22T22:25:16.760 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:16 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi067.rrokcu", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-04-22T22:25:16.760 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:16 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:25:16.760 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:16 smithi067 ceph-mon[21875]: Deploying daemon iscsi.foo.smithi067.rrokcu on smithi067 2024-04-22T22:25:18.758 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:18 smithi086 ceph-mon[22689]: pgmap v84: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 60 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:25:18.758 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:18 smithi086 ceph-mon[22689]: mgrmap e21: y(active, since 2m), standbys: x 2024-04-22T22:25:18.758 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:18 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:18.758 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:18 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:25:18.758 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:18 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:25:18.758 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:18 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:18.759 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:18 smithi067 ceph-mon[21875]: pgmap v84: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 60 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:25:18.759 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:18 smithi067 ceph-mon[21875]: mgrmap e21: y(active, since 2m), standbys: x 2024-04-22T22:25:18.759 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:18 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:18.759 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:18 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:25:18.759 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:18 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:25:18.759 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:18 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:18.759 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:18 smithi067 ceph-mon[26633]: pgmap v84: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 60 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:25:18.759 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:18 smithi067 ceph-mon[26633]: mgrmap e21: y(active, since 2m), standbys: x 2024-04-22T22:25:18.759 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:18 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:18.760 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:18 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:25:18.760 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:18 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:25:18.760 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:18 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:19.758 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:19 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/3750558785' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T22:25:19.758 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:19 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/2280476888' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/121635657"}]: dispatch 2024-04-22T22:25:19.758 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:19 smithi086 ceph-mon[22689]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/121635657"}]: dispatch 2024-04-22T22:25:19.759 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:19 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/3750558785' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T22:25:19.759 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:19 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/2280476888' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/121635657"}]: dispatch 2024-04-22T22:25:19.759 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:19 smithi067 ceph-mon[21875]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/121635657"}]: dispatch 2024-04-22T22:25:19.759 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:19 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/3750558785' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-22T22:25:19.759 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:19 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/2280476888' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/121635657"}]: dispatch 2024-04-22T22:25:19.759 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:19 smithi067 ceph-mon[26633]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/121635657"}]: dispatch 2024-04-22T22:25:20.758 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:20 smithi086 ceph-mon[22689]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 5.2 KiB/s rd, 383 B/s wr, 6 op/s; 6 B/s, 0 objects/s recovering 2024-04-22T22:25:20.758 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:20 smithi086 ceph-mon[22689]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/121635657"}]': finished 2024-04-22T22:25:20.758 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:20 smithi086 ceph-mon[22689]: osdmap e80: 8 total, 8 up, 8 in 2024-04-22T22:25:20.758 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:20 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/2239663458' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/3240066061"}]: dispatch 2024-04-22T22:25:20.759 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:25:20 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:25:20] "GET /metrics HTTP/1.1" 200 207245 "" "Prometheus/2.33.4" 2024-04-22T22:25:20.759 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:20 smithi067 ceph-mon[21875]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 5.2 KiB/s rd, 383 B/s wr, 6 op/s; 6 B/s, 0 objects/s recovering 2024-04-22T22:25:20.759 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:20 smithi067 ceph-mon[21875]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/121635657"}]': finished 2024-04-22T22:25:20.759 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:20 smithi067 ceph-mon[21875]: osdmap e80: 8 total, 8 up, 8 in 2024-04-22T22:25:20.759 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:20 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/2239663458' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/3240066061"}]: dispatch 2024-04-22T22:25:20.759 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:20 smithi067 ceph-mon[26633]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 5.2 KiB/s rd, 383 B/s wr, 6 op/s; 6 B/s, 0 objects/s recovering 2024-04-22T22:25:20.759 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:20 smithi067 ceph-mon[26633]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/121635657"}]': finished 2024-04-22T22:25:20.759 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:20 smithi067 ceph-mon[26633]: osdmap e80: 8 total, 8 up, 8 in 2024-04-22T22:25:20.760 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:20 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/2239663458' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/3240066061"}]: dispatch 2024-04-22T22:25:21.686 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:25:21 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:25:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:25:21.877 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:21 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/2239663458' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/3240066061"}]': finished 2024-04-22T22:25:21.877 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:21 smithi067 ceph-mon[21875]: osdmap e81: 8 total, 8 up, 8 in 2024-04-22T22:25:21.878 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:21 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/3820145467' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6801/288040075"}]: dispatch 2024-04-22T22:25:21.878 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:21 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:21.878 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:21 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/2239663458' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/3240066061"}]': finished 2024-04-22T22:25:21.878 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:21 smithi067 ceph-mon[26633]: osdmap e81: 8 total, 8 up, 8 in 2024-04-22T22:25:21.878 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:21 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/3820145467' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6801/288040075"}]: dispatch 2024-04-22T22:25:21.878 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:21 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:22.007 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:21 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/2239663458' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/3240066061"}]': finished 2024-04-22T22:25:22.007 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:21 smithi086 ceph-mon[22689]: osdmap e81: 8 total, 8 up, 8 in 2024-04-22T22:25:22.008 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:21 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/3820145467' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6801/288040075"}]: dispatch 2024-04-22T22:25:22.008 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:21 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:22.759 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[26633]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 6.4 KiB/s rd, 459 B/s wr, 8 op/s; 8 B/s, 0 objects/s recovering 2024-04-22T22:25:22.759 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/3820145467' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6801/288040075"}]': finished 2024-04-22T22:25:22.760 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[26633]: osdmap e82: 8 total, 8 up, 8 in 2024-04-22T22:25:22.760 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:22.760 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:22.760 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T22:25:22.760 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[26633]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T22:25:22.760 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[26633]: Adding iSCSI gateway http://:@172.21.15.67:5000 to Dashboard 2024-04-22T22:25:22.760 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T22:25:22.760 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[26633]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T22:25:22.760 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:22.760 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi067"}]: dispatch 2024-04-22T22:25:22.760 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[26633]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi067"}]: dispatch 2024-04-22T22:25:22.760 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:22.760 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:25:22.760 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:25:22.760 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[26633]: Checking dashboard <-> RGW credentials 2024-04-22T22:25:22.761 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/3145015469' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/910784755"}]: dispatch 2024-04-22T22:25:22.761 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[26633]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/910784755"}]: dispatch 2024-04-22T22:25:22.761 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:22.761 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[21875]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 6.4 KiB/s rd, 459 B/s wr, 8 op/s; 8 B/s, 0 objects/s recovering 2024-04-22T22:25:22.761 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/3820145467' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6801/288040075"}]': finished 2024-04-22T22:25:22.761 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[21875]: osdmap e82: 8 total, 8 up, 8 in 2024-04-22T22:25:22.761 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:22.761 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:22.761 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T22:25:22.761 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[21875]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T22:25:22.761 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[21875]: Adding iSCSI gateway http://:@172.21.15.67:5000 to Dashboard 2024-04-22T22:25:22.762 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T22:25:22.762 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[21875]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T22:25:22.762 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:22.762 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi067"}]: dispatch 2024-04-22T22:25:22.762 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[21875]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi067"}]: dispatch 2024-04-22T22:25:22.762 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:22.762 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:25:22.762 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:25:22.762 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[21875]: Checking dashboard <-> RGW credentials 2024-04-22T22:25:22.762 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/3145015469' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/910784755"}]: dispatch 2024-04-22T22:25:22.762 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[21875]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/910784755"}]: dispatch 2024-04-22T22:25:22.762 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:22 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:23.008 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:22 smithi086 ceph-mon[22689]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 6.4 KiB/s rd, 459 B/s wr, 8 op/s; 8 B/s, 0 objects/s recovering 2024-04-22T22:25:23.008 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:22 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/3820145467' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6801/288040075"}]': finished 2024-04-22T22:25:23.008 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:22 smithi086 ceph-mon[22689]: osdmap e82: 8 total, 8 up, 8 in 2024-04-22T22:25:23.008 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:22 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:23.008 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:22 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:23.008 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:22 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T22:25:23.008 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:22 smithi086 ceph-mon[22689]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-22T22:25:23.009 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:22 smithi086 ceph-mon[22689]: Adding iSCSI gateway http://:@172.21.15.67:5000 to Dashboard 2024-04-22T22:25:23.009 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:22 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T22:25:23.009 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:22 smithi086 ceph-mon[22689]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-22T22:25:23.009 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:22 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:23.009 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:22 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi067"}]: dispatch 2024-04-22T22:25:23.009 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:22 smithi086 ceph-mon[22689]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi067"}]: dispatch 2024-04-22T22:25:23.009 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:22 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:23.009 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:22 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:25:23.009 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:22 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:25:23.009 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:22 smithi086 ceph-mon[22689]: Checking dashboard <-> RGW credentials 2024-04-22T22:25:23.009 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:22 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/3145015469' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/910784755"}]: dispatch 2024-04-22T22:25:23.009 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:22 smithi086 ceph-mon[22689]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/910784755"}]: dispatch 2024-04-22T22:25:23.009 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:22 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:25:24.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:23 smithi086 ceph-mon[22689]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/910784755"}]': finished 2024-04-22T22:25:24.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:23 smithi086 ceph-mon[22689]: osdmap e83: 8 total, 8 up, 8 in 2024-04-22T22:25:24.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:23 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/2607198876' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/1267963091"}]: dispatch 2024-04-22T22:25:24.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:23 smithi086 ceph-mon[22689]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/1267963091"}]: dispatch 2024-04-22T22:25:24.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:23 smithi067 ceph-mon[21875]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/910784755"}]': finished 2024-04-22T22:25:24.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:23 smithi067 ceph-mon[21875]: osdmap e83: 8 total, 8 up, 8 in 2024-04-22T22:25:24.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:23 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/2607198876' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/1267963091"}]: dispatch 2024-04-22T22:25:24.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:23 smithi067 ceph-mon[21875]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/1267963091"}]: dispatch 2024-04-22T22:25:24.259 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:23 smithi067 ceph-mon[26633]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/910784755"}]': finished 2024-04-22T22:25:24.260 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:23 smithi067 ceph-mon[26633]: osdmap e83: 8 total, 8 up, 8 in 2024-04-22T22:25:24.260 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:23 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/2607198876' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/1267963091"}]: dispatch 2024-04-22T22:25:24.260 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:23 smithi067 ceph-mon[26633]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/1267963091"}]: dispatch 2024-04-22T22:25:25.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:24 smithi086 ceph-mon[22689]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:25:25.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:24 smithi086 ceph-mon[22689]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/1267963091"}]': finished 2024-04-22T22:25:25.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:24 smithi086 ceph-mon[22689]: osdmap e84: 8 total, 8 up, 8 in 2024-04-22T22:25:25.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:24 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/763703099' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/4134751820"}]: dispatch 2024-04-22T22:25:25.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:24 smithi086 ceph-mon[22689]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/4134751820"}]: dispatch 2024-04-22T22:25:25.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:24 smithi067 ceph-mon[21875]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:25:25.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:24 smithi067 ceph-mon[21875]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/1267963091"}]': finished 2024-04-22T22:25:25.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:24 smithi067 ceph-mon[21875]: osdmap e84: 8 total, 8 up, 8 in 2024-04-22T22:25:25.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:24 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/763703099' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/4134751820"}]: dispatch 2024-04-22T22:25:25.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:24 smithi067 ceph-mon[21875]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/4134751820"}]: dispatch 2024-04-22T22:25:25.259 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:24 smithi067 ceph-mon[26633]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:25:25.259 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:24 smithi067 ceph-mon[26633]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/1267963091"}]': finished 2024-04-22T22:25:25.259 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:24 smithi067 ceph-mon[26633]: osdmap e84: 8 total, 8 up, 8 in 2024-04-22T22:25:25.260 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:24 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/763703099' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/4134751820"}]: dispatch 2024-04-22T22:25:25.260 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:24 smithi067 ceph-mon[26633]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/4134751820"}]: dispatch 2024-04-22T22:25:26.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:25 smithi086 ceph-mon[22689]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/4134751820"}]': finished 2024-04-22T22:25:26.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:25 smithi086 ceph-mon[22689]: osdmap e85: 8 total, 8 up, 8 in 2024-04-22T22:25:26.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:25 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/572852323' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6801/4250787876"}]: dispatch 2024-04-22T22:25:26.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:25 smithi086 ceph-mon[22689]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6801/4250787876"}]: dispatch 2024-04-22T22:25:26.259 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:25 smithi067 ceph-mon[26633]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/4134751820"}]': finished 2024-04-22T22:25:26.259 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:25 smithi067 ceph-mon[26633]: osdmap e85: 8 total, 8 up, 8 in 2024-04-22T22:25:26.259 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:25 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/572852323' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6801/4250787876"}]: dispatch 2024-04-22T22:25:26.259 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:25 smithi067 ceph-mon[26633]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6801/4250787876"}]: dispatch 2024-04-22T22:25:26.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:25 smithi067 ceph-mon[21875]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/4134751820"}]': finished 2024-04-22T22:25:26.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:25 smithi067 ceph-mon[21875]: osdmap e85: 8 total, 8 up, 8 in 2024-04-22T22:25:26.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:25 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/572852323' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6801/4250787876"}]: dispatch 2024-04-22T22:25:26.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:25 smithi067 ceph-mon[21875]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6801/4250787876"}]: dispatch 2024-04-22T22:25:27.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:26 smithi086 ceph-mon[22689]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s 2024-04-22T22:25:27.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:26 smithi086 ceph-mon[22689]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6801/4250787876"}]': finished 2024-04-22T22:25:27.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:26 smithi086 ceph-mon[22689]: osdmap e86: 8 total, 8 up, 8 in 2024-04-22T22:25:27.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:26 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/1830694522' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/2868224087"}]: dispatch 2024-04-22T22:25:27.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:26 smithi067 ceph-mon[21875]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s 2024-04-22T22:25:27.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:26 smithi067 ceph-mon[21875]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6801/4250787876"}]': finished 2024-04-22T22:25:27.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:26 smithi067 ceph-mon[21875]: osdmap e86: 8 total, 8 up, 8 in 2024-04-22T22:25:27.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:26 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/1830694522' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/2868224087"}]: dispatch 2024-04-22T22:25:27.259 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:26 smithi067 ceph-mon[26633]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s 2024-04-22T22:25:27.259 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:26 smithi067 ceph-mon[26633]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6801/4250787876"}]': finished 2024-04-22T22:25:27.259 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:26 smithi067 ceph-mon[26633]: osdmap e86: 8 total, 8 up, 8 in 2024-04-22T22:25:27.259 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:26 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/1830694522' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/2868224087"}]: dispatch 2024-04-22T22:25:28.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:27 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/1830694522' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/2868224087"}]': finished 2024-04-22T22:25:28.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:27 smithi086 ceph-mon[22689]: osdmap e87: 8 total, 8 up, 8 in 2024-04-22T22:25:28.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:27 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/4080049986' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/3376956307"}]: dispatch 2024-04-22T22:25:28.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:27 smithi086 ceph-mon[22689]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/3376956307"}]: dispatch 2024-04-22T22:25:28.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:27 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/1830694522' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/2868224087"}]': finished 2024-04-22T22:25:28.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:27 smithi067 ceph-mon[21875]: osdmap e87: 8 total, 8 up, 8 in 2024-04-22T22:25:28.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:27 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/4080049986' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/3376956307"}]: dispatch 2024-04-22T22:25:28.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:27 smithi067 ceph-mon[21875]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/3376956307"}]: dispatch 2024-04-22T22:25:28.259 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:27 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/1830694522' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/2868224087"}]': finished 2024-04-22T22:25:28.259 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:27 smithi067 ceph-mon[26633]: osdmap e87: 8 total, 8 up, 8 in 2024-04-22T22:25:28.259 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:27 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/4080049986' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/3376956307"}]: dispatch 2024-04-22T22:25:28.260 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:27 smithi067 ceph-mon[26633]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/3376956307"}]: dispatch 2024-04-22T22:25:29.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:28 smithi086 ceph-mon[22689]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s 2024-04-22T22:25:29.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:28 smithi086 ceph-mon[22689]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/3376956307"}]': finished 2024-04-22T22:25:29.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:28 smithi086 ceph-mon[22689]: osdmap e88: 8 total, 8 up, 8 in 2024-04-22T22:25:29.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:28 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/2329948180' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/3520310390"}]: dispatch 2024-04-22T22:25:29.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:28 smithi067 ceph-mon[21875]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s 2024-04-22T22:25:29.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:28 smithi067 ceph-mon[21875]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/3376956307"}]': finished 2024-04-22T22:25:29.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:28 smithi067 ceph-mon[21875]: osdmap e88: 8 total, 8 up, 8 in 2024-04-22T22:25:29.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:28 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/2329948180' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/3520310390"}]: dispatch 2024-04-22T22:25:29.259 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:28 smithi067 ceph-mon[26633]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s 2024-04-22T22:25:29.260 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:28 smithi067 ceph-mon[26633]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/3376956307"}]': finished 2024-04-22T22:25:29.260 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:28 smithi067 ceph-mon[26633]: osdmap e88: 8 total, 8 up, 8 in 2024-04-22T22:25:29.260 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:28 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/2329948180' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/3520310390"}]: dispatch 2024-04-22T22:25:30.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:29 smithi086 ceph-mon[22689]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:25:30.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:29 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/2329948180' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/3520310390"}]': finished 2024-04-22T22:25:30.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:29 smithi086 ceph-mon[22689]: osdmap e89: 8 total, 8 up, 8 in 2024-04-22T22:25:30.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:29 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/2962396880' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6800/1447658117"}]: dispatch 2024-04-22T22:25:30.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:29 smithi086 ceph-mon[22689]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6800/1447658117"}]: dispatch 2024-04-22T22:25:30.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:29 smithi067 ceph-mon[21875]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:25:30.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:29 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/2329948180' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/3520310390"}]': finished 2024-04-22T22:25:30.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:29 smithi067 ceph-mon[21875]: osdmap e89: 8 total, 8 up, 8 in 2024-04-22T22:25:30.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:29 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/2962396880' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6800/1447658117"}]: dispatch 2024-04-22T22:25:30.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:29 smithi067 ceph-mon[21875]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6800/1447658117"}]: dispatch 2024-04-22T22:25:30.259 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:29 smithi067 ceph-mon[26633]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:25:30.259 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:29 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/2329948180' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/3520310390"}]': finished 2024-04-22T22:25:30.259 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:29 smithi067 ceph-mon[26633]: osdmap e89: 8 total, 8 up, 8 in 2024-04-22T22:25:30.259 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:29 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/2962396880' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6800/1447658117"}]: dispatch 2024-04-22T22:25:30.260 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:29 smithi067 ceph-mon[26633]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6800/1447658117"}]: dispatch 2024-04-22T22:25:30.981 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:25:30 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:25:30] "GET /metrics HTTP/1.1" 200 208085 "" "Prometheus/2.33.4" 2024-04-22T22:25:31.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:30 smithi086 ceph-mon[22689]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:25:31.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:30 smithi086 ceph-mon[22689]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6800/1447658117"}]': finished 2024-04-22T22:25:31.259 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:30 smithi086 ceph-mon[22689]: osdmap e90: 8 total, 8 up, 8 in 2024-04-22T22:25:31.259 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:30 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/2391442483' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/204413874"}]: dispatch 2024-04-22T22:25:31.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:30 smithi067 ceph-mon[21875]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:25:31.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:30 smithi067 ceph-mon[21875]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6800/1447658117"}]': finished 2024-04-22T22:25:31.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:30 smithi067 ceph-mon[21875]: osdmap e90: 8 total, 8 up, 8 in 2024-04-22T22:25:31.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:30 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/2391442483' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/204413874"}]: dispatch 2024-04-22T22:25:31.260 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:30 smithi067 ceph-mon[26633]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:25:31.260 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:30 smithi067 ceph-mon[26633]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6800/1447658117"}]': finished 2024-04-22T22:25:31.260 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:30 smithi067 ceph-mon[26633]: osdmap e90: 8 total, 8 up, 8 in 2024-04-22T22:25:31.260 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:30 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/2391442483' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/204413874"}]: dispatch 2024-04-22T22:25:31.758 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:25:31 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:25:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:25:32.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:31 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/2391442483' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/204413874"}]': finished 2024-04-22T22:25:32.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:31 smithi086 ceph-mon[22689]: osdmap e91: 8 total, 8 up, 8 in 2024-04-22T22:25:32.259 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:31 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/2380705764' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6800/288040075"}]: dispatch 2024-04-22T22:25:32.259 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:31 smithi086 ceph-mon[22689]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6800/288040075"}]: dispatch 2024-04-22T22:25:32.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:31 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/2391442483' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/204413874"}]': finished 2024-04-22T22:25:32.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:31 smithi067 ceph-mon[21875]: osdmap e91: 8 total, 8 up, 8 in 2024-04-22T22:25:32.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:31 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/2380705764' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6800/288040075"}]: dispatch 2024-04-22T22:25:32.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:31 smithi067 ceph-mon[21875]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6800/288040075"}]: dispatch 2024-04-22T22:25:32.260 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:31 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/2391442483' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/204413874"}]': finished 2024-04-22T22:25:32.260 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:31 smithi067 ceph-mon[26633]: osdmap e91: 8 total, 8 up, 8 in 2024-04-22T22:25:32.260 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:31 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/2380705764' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6800/288040075"}]: dispatch 2024-04-22T22:25:32.260 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:31 smithi067 ceph-mon[26633]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6800/288040075"}]: dispatch 2024-04-22T22:25:33.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:33 smithi086 ceph-mon[22689]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:25:33.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:33 smithi086 ceph-mon[22689]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6800/288040075"}]': finished 2024-04-22T22:25:33.259 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:33 smithi086 ceph-mon[22689]: osdmap e92: 8 total, 8 up, 8 in 2024-04-22T22:25:33.259 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:33 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/1545293419' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/2710747697"}]: dispatch 2024-04-22T22:25:33.259 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:33 smithi086 ceph-mon[22689]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/2710747697"}]: dispatch 2024-04-22T22:25:33.259 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:33 smithi067 ceph-mon[26633]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:25:33.259 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:33 smithi067 ceph-mon[26633]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6800/288040075"}]': finished 2024-04-22T22:25:33.259 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:33 smithi067 ceph-mon[26633]: osdmap e92: 8 total, 8 up, 8 in 2024-04-22T22:25:33.259 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:33 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/1545293419' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/2710747697"}]: dispatch 2024-04-22T22:25:33.259 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:33 smithi067 ceph-mon[26633]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/2710747697"}]: dispatch 2024-04-22T22:25:33.260 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:33 smithi067 ceph-mon[21875]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:25:33.260 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:33 smithi067 ceph-mon[21875]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6800/288040075"}]': finished 2024-04-22T22:25:33.260 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:33 smithi067 ceph-mon[21875]: osdmap e92: 8 total, 8 up, 8 in 2024-04-22T22:25:33.260 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:33 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/1545293419' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/2710747697"}]: dispatch 2024-04-22T22:25:33.260 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:33 smithi067 ceph-mon[21875]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/2710747697"}]: dispatch 2024-04-22T22:25:34.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:34 smithi086 ceph-mon[22689]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/2710747697"}]': finished 2024-04-22T22:25:34.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:34 smithi086 ceph-mon[22689]: osdmap e93: 8 total, 8 up, 8 in 2024-04-22T22:25:34.258 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:34 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/12917207' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6800/4250787876"}]: dispatch 2024-04-22T22:25:34.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:34 smithi067 ceph-mon[21875]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/2710747697"}]': finished 2024-04-22T22:25:34.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:34 smithi067 ceph-mon[21875]: osdmap e93: 8 total, 8 up, 8 in 2024-04-22T22:25:34.259 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:34 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/12917207' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6800/4250787876"}]: dispatch 2024-04-22T22:25:34.259 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:34 smithi067 ceph-mon[26633]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:0/2710747697"}]': finished 2024-04-22T22:25:34.259 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:34 smithi067 ceph-mon[26633]: osdmap e93: 8 total, 8 up, 8 in 2024-04-22T22:25:34.259 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:34 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/12917207' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6800/4250787876"}]: dispatch 2024-04-22T22:25:35.508 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:35 smithi086 ceph-mon[22689]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:25:35.508 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:35 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/12917207' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6800/4250787876"}]': finished 2024-04-22T22:25:35.508 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:35 smithi086 ceph-mon[22689]: osdmap e94: 8 total, 8 up, 8 in 2024-04-22T22:25:35.508 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:35 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/3425328363' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6801/1447658117"}]: dispatch 2024-04-22T22:25:35.508 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:35 smithi086 ceph-mon[22689]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6801/1447658117"}]: dispatch 2024-04-22T22:25:35.509 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:35 smithi067 ceph-mon[21875]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:25:35.509 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:35 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/12917207' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6800/4250787876"}]': finished 2024-04-22T22:25:35.509 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:35 smithi067 ceph-mon[21875]: osdmap e94: 8 total, 8 up, 8 in 2024-04-22T22:25:35.509 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:35 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/3425328363' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6801/1447658117"}]: dispatch 2024-04-22T22:25:35.510 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:35 smithi067 ceph-mon[21875]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6801/1447658117"}]: dispatch 2024-04-22T22:25:35.510 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:35 smithi067 ceph-mon[26633]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:25:35.510 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:35 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/12917207' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6800/4250787876"}]': finished 2024-04-22T22:25:35.510 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:35 smithi067 ceph-mon[26633]: osdmap e94: 8 total, 8 up, 8 in 2024-04-22T22:25:35.510 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:35 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/3425328363' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6801/1447658117"}]: dispatch 2024-04-22T22:25:35.510 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:35 smithi067 ceph-mon[26633]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6801/1447658117"}]: dispatch 2024-04-22T22:25:36.508 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:36 smithi086 ceph-mon[22689]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6801/1447658117"}]': finished 2024-04-22T22:25:36.508 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:36 smithi086 ceph-mon[22689]: osdmap e95: 8 total, 8 up, 8 in 2024-04-22T22:25:36.509 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:36 smithi067 ceph-mon[21875]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6801/1447658117"}]': finished 2024-04-22T22:25:36.509 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:36 smithi067 ceph-mon[21875]: osdmap e95: 8 total, 8 up, 8 in 2024-04-22T22:25:36.509 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:36 smithi067 ceph-mon[26633]: from='client.? ' entity='client.iscsi.foo.smithi067.rrokcu' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.67:6801/1447658117"}]': finished 2024-04-22T22:25:36.510 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:36 smithi067 ceph-mon[26633]: osdmap e95: 8 total, 8 up, 8 in 2024-04-22T22:25:37.508 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:37 smithi086 ceph-mon[22689]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:25:37.509 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:37 smithi067 ceph-mon[21875]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:25:37.510 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:37 smithi067 ceph-mon[26633]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:25:39.508 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:39 smithi086 ceph-mon[22689]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 951 B/s rd, 0 op/s 2024-04-22T22:25:39.509 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:39 smithi067 ceph-mon[21875]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 951 B/s rd, 0 op/s 2024-04-22T22:25:39.509 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:39 smithi067 ceph-mon[26633]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 951 B/s rd, 0 op/s 2024-04-22T22:25:40.508 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:40 smithi086 ceph-mon[22689]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:25:40.509 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:40 smithi067 ceph-mon[21875]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:25:40.510 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:40 smithi067 ceph-mon[26633]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:25:41.010 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:25:40 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:25:40] "GET /metrics HTTP/1.1" 200 208085 "" "Prometheus/2.33.4" 2024-04-22T22:25:41.409 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:41 smithi086 ceph-mon[22689]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-04-22T22:25:41.509 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:41 smithi067 ceph-mon[21875]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-04-22T22:25:41.510 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:41 smithi067 ceph-mon[26633]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-04-22T22:25:41.758 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:25:41 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:25:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:25:43.508 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:43 smithi086 ceph-mon[22689]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:25:43.509 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:43 smithi067 ceph-mon[21875]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:25:43.509 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:43 smithi067 ceph-mon[26633]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:25:45.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:45 smithi086 ceph-mon[22689]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T22:25:45.509 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:45 smithi067 ceph-mon[21875]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T22:25:45.509 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:45 smithi067 ceph-mon[26633]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-22T22:25:47.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:47 smithi086 ceph-mon[22689]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 987 B/s rd, 0 op/s 2024-04-22T22:25:47.509 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:47 smithi067 ceph-mon[21875]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 987 B/s rd, 0 op/s 2024-04-22T22:25:47.510 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:47 smithi067 ceph-mon[26633]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 987 B/s rd, 0 op/s 2024-04-22T22:25:49.508 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:49 smithi086 ceph-mon[22689]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:25:49.509 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:49 smithi067 ceph-mon[21875]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:25:49.509 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:49 smithi067 ceph-mon[26633]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:25:50.508 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:50 smithi086 ceph-mon[22689]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:25:50.510 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:50 smithi067 ceph-mon[21875]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:25:50.510 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:50 smithi067 ceph-mon[26633]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:25:51.010 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:25:50 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:25:50] "GET /metrics HTTP/1.1" 200 208076 "" "Prometheus/2.33.4" 2024-04-22T22:25:51.413 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:51 smithi086 ceph-mon[22689]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:25:51.509 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:51 smithi067 ceph-mon[21875]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:25:51.510 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:51 smithi067 ceph-mon[26633]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:25:51.758 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:25:51 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:25:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:25:53.508 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:53 smithi086 ceph-mon[22689]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:25:53.509 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:53 smithi067 ceph-mon[21875]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:25:53.510 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:53 smithi067 ceph-mon[26633]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:25:55.508 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:55 smithi086 ceph-mon[22689]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:25:55.510 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:55 smithi067 ceph-mon[21875]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:25:55.510 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:55 smithi067 ceph-mon[26633]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:25:57.508 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:57 smithi086 ceph-mon[22689]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:25:57.510 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:57 smithi067 ceph-mon[21875]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:25:57.510 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:57 smithi067 ceph-mon[26633]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:25:59.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:25:59 smithi086 ceph-mon[22689]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:25:59.510 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:25:59 smithi067 ceph-mon[21875]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:25:59.510 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:25:59 smithi067 ceph-mon[26633]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:00.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:00 smithi086 ceph-mon[22689]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:26:00.509 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:00 smithi067 ceph-mon[21875]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:26:00.510 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:00 smithi067 ceph-mon[26633]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:26:01.010 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:26:00 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:26:00] "GET /metrics HTTP/1.1" 200 208028 "" "Prometheus/2.33.4" 2024-04-22T22:26:01.414 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:01 smithi086 ceph-mon[22689]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:26:01.510 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:01 smithi067 ceph-mon[21875]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:26:01.510 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:01 smithi067 ceph-mon[26633]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:26:01.759 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:26:01 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:26:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:26:03.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:03 smithi086 ceph-mon[22689]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:03.510 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:03 smithi067 ceph-mon[21875]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:03.511 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:03 smithi067 ceph-mon[26633]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:05.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:05 smithi086 ceph-mon[22689]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:05.510 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:05 smithi067 ceph-mon[21875]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:05.510 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:05 smithi067 ceph-mon[26633]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:07.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:07 smithi086 ceph-mon[22689]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:26:07.510 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:07 smithi067 ceph-mon[21875]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:26:07.510 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:07 smithi067 ceph-mon[26633]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:26:09.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:09 smithi086 ceph-mon[22689]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:09.510 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:09 smithi067 ceph-mon[21875]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:09.510 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:09 smithi067 ceph-mon[26633]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:10.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:10 smithi086 ceph-mon[22689]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:26:10.510 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:10 smithi067 ceph-mon[21875]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:26:10.510 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:10 smithi067 ceph-mon[26633]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:26:11.010 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:26:10 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:26:10] "GET /metrics HTTP/1.1" 200 208028 "" "Prometheus/2.33.4" 2024-04-22T22:26:11.410 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:11 smithi086 ceph-mon[22689]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:26:11.510 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:11 smithi067 ceph-mon[21875]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:26:11.510 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:11 smithi067 ceph-mon[26633]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:26:11.759 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:26:11 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:26:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:26:13.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:13 smithi086 ceph-mon[22689]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:13.510 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:13 smithi067 ceph-mon[21875]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:13.510 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:13 smithi067 ceph-mon[26633]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:14.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:14 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:26:14.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:14 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:26:14.510 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:14 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:26:14.510 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:14 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:26:14.510 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:14 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:26:14.510 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:14 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:26:15.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:15 smithi086 ceph-mon[22689]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:15.510 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:15 smithi067 ceph-mon[21875]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:15.510 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:15 smithi067 ceph-mon[26633]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:17.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:17 smithi086 ceph-mon[22689]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:26:17.510 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:17 smithi067 ceph-mon[21875]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:26:17.510 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:17 smithi067 ceph-mon[26633]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:26:19.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:19 smithi086 ceph-mon[22689]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:19.510 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:19 smithi067 ceph-mon[21875]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:19.510 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:19 smithi067 ceph-mon[26633]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:20.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:20 smithi086 ceph-mon[22689]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:26:20.510 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:20 smithi067 ceph-mon[21875]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:26:20.510 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:20 smithi067 ceph-mon[26633]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:26:21.010 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:26:20 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:26:20] "GET /metrics HTTP/1.1" 200 207984 "" "Prometheus/2.33.4" 2024-04-22T22:26:21.414 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:21 smithi086 ceph-mon[22689]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:26:21.510 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:21 smithi067 ceph-mon[21875]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:26:21.511 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:21 smithi067 ceph-mon[26633]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:26:21.759 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:26:21 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:26:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:26:23.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:23 smithi086 ceph-mon[22689]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:23.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:23 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:26:23.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:23 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:26:23.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:23 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:26:23.510 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:23 smithi067 ceph-mon[21875]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:23.510 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:23 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:26:23.510 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:23 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:26:23.510 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:23 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:26:23.510 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:23 smithi067 ceph-mon[26633]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:23.511 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:23 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:26:23.511 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:23 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:26:23.511 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:23 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:26:25.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:25 smithi086 ceph-mon[22689]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:25.510 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:25 smithi067 ceph-mon[21875]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:25.510 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:25 smithi067 ceph-mon[26633]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:27.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:27 smithi086 ceph-mon[22689]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:26:27.510 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:27 smithi067 ceph-mon[21875]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:26:27.510 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:27 smithi067 ceph-mon[26633]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:26:29.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:29 smithi086 ceph-mon[22689]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:29.510 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:29 smithi067 ceph-mon[21875]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:29.510 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:29 smithi067 ceph-mon[26633]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:30.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:30 smithi086 ceph-mon[22689]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:26:30.510 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:30 smithi067 ceph-mon[21875]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:26:30.510 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:30 smithi067 ceph-mon[26633]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:26:31.010 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:26:30 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:26:30] "GET /metrics HTTP/1.1" 200 207997 "" "Prometheus/2.33.4" 2024-04-22T22:26:31.415 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:31 smithi086 ceph-mon[22689]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:26:31.510 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:31 smithi067 ceph-mon[21875]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:26:31.511 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:31 smithi067 ceph-mon[26633]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:26:31.759 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:26:31 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:26:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:26:33.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:33 smithi086 ceph-mon[22689]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:33.511 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:33 smithi067 ceph-mon[21875]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:33.511 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:33 smithi067 ceph-mon[26633]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:35.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:35 smithi086 ceph-mon[22689]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:35.510 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:35 smithi067 ceph-mon[21875]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:35.511 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:35 smithi067 ceph-mon[26633]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:37.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:37 smithi086 ceph-mon[22689]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:26:37.511 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:37 smithi067 ceph-mon[21875]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:26:37.511 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:37 smithi067 ceph-mon[26633]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:26:39.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:39 smithi086 ceph-mon[22689]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:39.511 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:39 smithi067 ceph-mon[21875]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:39.511 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:39 smithi067 ceph-mon[26633]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:40.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:40 smithi086 ceph-mon[22689]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:26:40.510 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:40 smithi067 ceph-mon[21875]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:26:40.511 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:40 smithi067 ceph-mon[26633]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:26:41.011 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:26:40 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:26:40] "GET /metrics HTTP/1.1" 200 207997 "" "Prometheus/2.33.4" 2024-04-22T22:26:41.411 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:41 smithi086 ceph-mon[22689]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:26:41.511 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:41 smithi067 ceph-mon[21875]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:26:41.511 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:41 smithi067 ceph-mon[26633]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:26:41.759 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:26:41 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:26:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:26:43.510 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:43 smithi086 ceph-mon[22689]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:43.510 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:43 smithi067 ceph-mon[21875]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:43.511 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:43 smithi067 ceph-mon[26633]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:45.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:45 smithi086 ceph-mon[22689]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:45.510 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:45 smithi067 ceph-mon[21875]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:45.511 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:45 smithi067 ceph-mon[26633]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:47.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:47 smithi086 ceph-mon[22689]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:26:47.511 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:47 smithi067 ceph-mon[21875]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:26:47.511 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:47 smithi067 ceph-mon[26633]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:26:49.509 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:49 smithi086 ceph-mon[22689]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:49.511 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:49 smithi067 ceph-mon[21875]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:49.511 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:49 smithi067 ceph-mon[26633]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:50.510 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:50 smithi086 ceph-mon[22689]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:26:50.511 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:50 smithi067 ceph-mon[21875]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:26:50.511 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:50 smithi067 ceph-mon[26633]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:26:51.011 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:26:50 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:26:50] "GET /metrics HTTP/1.1" 200 207988 "" "Prometheus/2.33.4" 2024-04-22T22:26:51.264 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:51 smithi067 ceph-mon[21875]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:26:51.264 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:51 smithi067 ceph-mon[26633]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:26:51.510 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:51 smithi086 ceph-mon[22689]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:26:51.510 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:26:51 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:26:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:26:53.510 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:53 smithi086 ceph-mon[22689]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:53.511 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:53 smithi067 ceph-mon[21875]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:53.511 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:53 smithi067 ceph-mon[26633]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:55.510 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:55 smithi086 ceph-mon[22689]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:55.511 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:55 smithi067 ceph-mon[21875]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:55.511 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:55 smithi067 ceph-mon[26633]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:57.510 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:57 smithi086 ceph-mon[22689]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:26:57.511 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:57 smithi067 ceph-mon[21875]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:26:57.511 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:57 smithi067 ceph-mon[26633]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:26:59.510 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:26:59 smithi086 ceph-mon[22689]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:59.511 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:26:59 smithi067 ceph-mon[21875]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:26:59.511 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:26:59 smithi067 ceph-mon[26633]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:00.510 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:00 smithi086 ceph-mon[22689]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:27:00.511 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:00 smithi067 ceph-mon[21875]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:27:00.511 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:00 smithi067 ceph-mon[26633]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:27:01.011 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:27:00 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:27:00] "GET /metrics HTTP/1.1" 200 207998 "" "Prometheus/2.33.4" 2024-04-22T22:27:01.510 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:01 smithi086 ceph-mon[22689]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:27:01.511 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:27:01 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:27:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:27:01.511 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:01 smithi067 ceph-mon[21875]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:27:01.512 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:01 smithi067 ceph-mon[26633]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:27:03.510 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:03 smithi086 ceph-mon[22689]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:03.511 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:03 smithi067 ceph-mon[21875]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:03.511 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:03 smithi067 ceph-mon[26633]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:05.510 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:05 smithi086 ceph-mon[22689]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:05.511 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:05 smithi067 ceph-mon[21875]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:05.511 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:05 smithi067 ceph-mon[26633]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:07.510 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:07 smithi086 ceph-mon[22689]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:27:07.511 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:07 smithi067 ceph-mon[21875]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:27:07.511 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:07 smithi067 ceph-mon[26633]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:27:09.510 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:09 smithi086 ceph-mon[22689]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:09.511 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:09 smithi067 ceph-mon[21875]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:09.511 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:09 smithi067 ceph-mon[26633]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:10.510 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:10 smithi086 ceph-mon[22689]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:27:10.511 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:10 smithi067 ceph-mon[21875]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:27:10.511 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:10 smithi067 ceph-mon[26633]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:27:11.011 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:27:10 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:27:10] "GET /metrics HTTP/1.1" 200 207998 "" "Prometheus/2.33.4" 2024-04-22T22:27:11.510 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:11 smithi086 ceph-mon[22689]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:27:11.510 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:27:11 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:27:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:27:11.511 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:11 smithi067 ceph-mon[21875]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:27:11.511 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:11 smithi067 ceph-mon[26633]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:27:13.510 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:13 smithi086 ceph-mon[22689]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:13.511 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:13 smithi067 ceph-mon[21875]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:13.512 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:13 smithi067 ceph-mon[26633]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:14.510 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:14 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:27:14.510 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:14 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:27:14.511 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:14 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:27:14.511 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:14 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:27:14.511 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:14 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:27:14.512 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:14 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:27:15.510 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:15 smithi086 ceph-mon[22689]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:15.511 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:15 smithi067 ceph-mon[21875]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:15.511 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:15 smithi067 ceph-mon[26633]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:17.510 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:17 smithi086 ceph-mon[22689]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:27:17.511 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:17 smithi067 ceph-mon[21875]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:27:17.511 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:17 smithi067 ceph-mon[26633]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:27:19.510 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:19 smithi086 ceph-mon[22689]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:19.511 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:19 smithi067 ceph-mon[21875]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:19.511 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:19 smithi067 ceph-mon[26633]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:20.510 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:20 smithi086 ceph-mon[22689]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:27:20.511 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:20 smithi067 ceph-mon[21875]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:27:20.512 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:20 smithi067 ceph-mon[26633]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:27:21.011 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:27:20 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:27:20] "GET /metrics HTTP/1.1" 200 207985 "" "Prometheus/2.33.4" 2024-04-22T22:27:21.510 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:21 smithi086 ceph-mon[22689]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:27:21.511 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:27:21 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:27:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:27:21.511 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:21 smithi067 ceph-mon[21875]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:27:21.511 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:21 smithi067 ceph-mon[26633]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:27:23.510 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:23 smithi086 ceph-mon[22689]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:23.511 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:23 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:27:23.511 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:23 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:27:23.511 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:23 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:27:23.511 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:23 smithi067 ceph-mon[21875]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:23.511 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:23 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:27:23.511 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:23 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:27:23.512 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:23 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:27:23.512 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:23 smithi067 ceph-mon[26633]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:23.512 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:23 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:27:23.512 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:23 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:27:23.512 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:23 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:27:25.510 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:25 smithi086 ceph-mon[22689]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:25.511 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:25 smithi067 ceph-mon[21875]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:25.511 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:25 smithi067 ceph-mon[26633]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:27.760 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:27 smithi086 ceph-mon[22689]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:27:27.761 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:27 smithi067 ceph-mon[21875]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:27:27.762 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:27 smithi067 ceph-mon[26633]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:27:29.760 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:29 smithi086 ceph-mon[22689]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:29.761 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:29 smithi067 ceph-mon[21875]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:29.762 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:29 smithi067 ceph-mon[26633]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:30.573 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:30 smithi067 ceph-mon[21875]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:27:30.573 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:30 smithi067 ceph-mon[26633]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:27:30.760 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:30 smithi086 ceph-mon[22689]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:27:31.012 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:27:30 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:27:30] "GET /metrics HTTP/1.1" 200 207983 "" "Prometheus/2.33.4" 2024-04-22T22:27:31.761 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:31 smithi086 ceph-mon[22689]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:27:31.761 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:27:31 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:27:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:27:31.762 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:31 smithi067 ceph-mon[21875]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:27:31.762 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:31 smithi067 ceph-mon[26633]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:27:33.760 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:33 smithi086 ceph-mon[22689]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:33.761 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:33 smithi067 ceph-mon[21875]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:33.762 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:33 smithi067 ceph-mon[26633]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:35.760 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:35 smithi086 ceph-mon[22689]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:35.761 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:35 smithi067 ceph-mon[21875]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:35.762 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:35 smithi067 ceph-mon[26633]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:37.531 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:37 smithi067 ceph-mon[21875]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:27:37.531 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:37 smithi067 ceph-mon[26633]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:27:37.761 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:37 smithi086 ceph-mon[22689]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:27:39.761 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:39 smithi086 ceph-mon[22689]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:39.762 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:39 smithi067 ceph-mon[21875]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:39.762 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:39 smithi067 ceph-mon[26633]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:40.573 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:40 smithi067 ceph-mon[21875]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:27:40.573 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:40 smithi067 ceph-mon[26633]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:27:40.760 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:40 smithi086 ceph-mon[22689]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:27:41.012 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:27:40 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:27:40] "GET /metrics HTTP/1.1" 200 207983 "" "Prometheus/2.33.4" 2024-04-22T22:27:41.662 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:41 smithi086 ceph-mon[22689]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:27:41.662 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:27:41 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:27:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:27:41.762 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:41 smithi067 ceph-mon[21875]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:27:41.762 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:41 smithi067 ceph-mon[26633]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:27:43.761 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:43 smithi086 ceph-mon[22689]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:43.762 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:43 smithi067 ceph-mon[21875]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:43.762 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:43 smithi067 ceph-mon[26633]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:45.761 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:45 smithi086 ceph-mon[22689]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:45.762 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:45 smithi067 ceph-mon[21875]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:45.762 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:45 smithi067 ceph-mon[26633]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:47.761 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:47 smithi086 ceph-mon[22689]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:27:47.762 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:47 smithi067 ceph-mon[21875]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:27:47.762 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:47 smithi067 ceph-mon[26633]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:27:49.761 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:49 smithi086 ceph-mon[22689]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:49.762 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:49 smithi067 ceph-mon[21875]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:49.762 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:49 smithi067 ceph-mon[26633]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:50.574 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:50 smithi067 ceph-mon[21875]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:27:50.574 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:50 smithi067 ceph-mon[26633]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:27:50.761 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:50 smithi086 ceph-mon[22689]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:27:51.012 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:27:50 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:27:50] "GET /metrics HTTP/1.1" 200 207951 "" "Prometheus/2.33.4" 2024-04-22T22:27:51.761 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:51 smithi086 ceph-mon[22689]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:27:51.761 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:27:51 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:27:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:27:51.762 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:51 smithi067 ceph-mon[21875]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:27:51.762 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:51 smithi067 ceph-mon[26633]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:27:53.761 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:53 smithi086 ceph-mon[22689]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:53.762 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:53 smithi067 ceph-mon[21875]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:53.762 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:53 smithi067 ceph-mon[26633]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:55.761 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:55 smithi086 ceph-mon[22689]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:55.762 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:55 smithi067 ceph-mon[21875]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:55.762 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:55 smithi067 ceph-mon[26633]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:57.761 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:57 smithi086 ceph-mon[22689]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:27:57.762 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:57 smithi067 ceph-mon[21875]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:27:57.762 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:57 smithi067 ceph-mon[26633]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:27:59.761 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:27:59 smithi086 ceph-mon[22689]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:59.762 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:27:59 smithi067 ceph-mon[21875]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:27:59.762 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:27:59 smithi067 ceph-mon[26633]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:00.761 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:00 smithi086 ceph-mon[22689]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:28:00.762 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:00 smithi067 ceph-mon[21875]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:28:00.762 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:00 smithi067 ceph-mon[26633]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:28:00.763 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:28:00 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:28:00] "GET /metrics HTTP/1.1" 200 207946 "" "Prometheus/2.33.4" 2024-04-22T22:28:01.761 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:01 smithi086 ceph-mon[22689]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:28:01.761 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:28:01 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:28:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:28:01.762 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:01 smithi067 ceph-mon[21875]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:28:01.762 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:01 smithi067 ceph-mon[26633]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:28:03.761 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:03 smithi086 ceph-mon[22689]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:03.762 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:03 smithi067 ceph-mon[21875]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:03.762 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:03 smithi067 ceph-mon[26633]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:05.761 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:05 smithi086 ceph-mon[22689]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:05.762 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:05 smithi067 ceph-mon[21875]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:05.762 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:05 smithi067 ceph-mon[26633]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:06.761 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:06 smithi086 ceph-mon[22689]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:28:06.762 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:06 smithi067 ceph-mon[21875]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:28:06.762 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:06 smithi067 ceph-mon[26633]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:28:08.761 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:08 smithi086 ceph-mon[22689]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:08.762 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:08 smithi067 ceph-mon[21875]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:08.762 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:08 smithi067 ceph-mon[26633]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:09.761 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:09 smithi086 ceph-mon[22689]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:28:09.762 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:09 smithi067 ceph-mon[21875]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:28:09.762 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:09 smithi067 ceph-mon[26633]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:28:11.011 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:10 smithi086 ceph-mon[22689]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:28:11.012 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:10 smithi067 ceph-mon[21875]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:28:11.012 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:10 smithi067 ceph-mon[26633]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:28:11.013 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:28:10 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:28:10] "GET /metrics HTTP/1.1" 200 207946 "" "Prometheus/2.33.4" 2024-04-22T22:28:11.761 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:28:11 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:28:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:28:13.011 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:12 smithi086 ceph-mon[22689]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:13.012 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:12 smithi067 ceph-mon[21875]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:13.012 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:12 smithi067 ceph-mon[26633]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:14.011 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:13 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:28:14.011 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:13 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:28:14.012 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:13 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:28:14.012 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:13 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:28:14.012 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:13 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:28:14.013 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:13 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:28:15.011 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:14 smithi086 ceph-mon[22689]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:15.012 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:14 smithi067 ceph-mon[21875]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:15.012 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:14 smithi067 ceph-mon[26633]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:16.058 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -e sha1=430e09df97c8fc7dc2b2ae424f68ed11366c540f -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-04-22T22:28:17.011 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:16 smithi086 ceph-mon[22689]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:28:17.012 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:16 smithi067 ceph-mon[21875]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:28:17.012 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:16 smithi067 ceph-mon[26633]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:28:18.763 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -e sha1=430e09df97c8fc7dc2b2ae424f68ed11366c540f -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-04-22T22:28:19.011 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:18 smithi086 ceph-mon[22689]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:19.012 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:18 smithi067 ceph-mon[21875]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:19.013 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:18 smithi067 ceph-mon[26633]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:20.011 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:19 smithi086 ceph-mon[22689]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:28:20.013 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:19 smithi067 ceph-mon[21875]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:28:20.013 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:19 smithi067 ceph-mon[26633]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:28:20.746 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:28:20 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:28:20] "GET /metrics HTTP/1.1" 200 207936 "" "Prometheus/2.33.4" 2024-04-22T22:28:21.011 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:20 smithi086 ceph-mon[22689]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:28:21.012 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:20 smithi067 ceph-mon[21875]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:28:21.012 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:20 smithi067 ceph-mon[26633]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:28:21.369 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -e sha1=430e09df97c8fc7dc2b2ae424f68ed11366c540f -- bash -c 'ceph config set global log_to_journald false --force' 2024-04-22T22:28:21.761 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:28:21 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:28:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:28:22.962 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:22 smithi067 ceph-mon[21875]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:22.962 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:22 smithi067 ceph-mon[26633]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:23.011 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:22 smithi086 ceph-mon[22689]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:24.012 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:23 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:28:24.012 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:23 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:28:24.012 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:23 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:28:24.013 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:23 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:28:24.013 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:23 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:28:24.013 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:23 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:28:24.013 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:23 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:28:24.013 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:23 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:28:24.013 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:23 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:28:24.049 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -e sha1=430e09df97c8fc7dc2b2ae424f68ed11366c540f -- bash -c 'ceph orch ps' 2024-04-22T22:28:25.543 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:25 smithi067 ceph-mon[21875]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:25.543 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:25 smithi067 ceph-mon[26633]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:25.761 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:25 smithi086 ceph-mon[22689]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:25.960 INFO:teuthology.orchestra.run.smithi067.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T22:28:25.960 INFO:teuthology.orchestra.run.smithi067.stdout:alertmanager.a smithi067 *:9093,9094 running (3m) 3m ago 4m 17.6M - ba2b418f427c 9d0e1f18c1cb 2024-04-22T22:28:25.960 INFO:teuthology.orchestra.run.smithi067.stdout:grafana.a smithi086 *:3000 running (3m) 3m ago 3m 47.6M - 8.3.5 dad864ee21e9 5d6c83f8480c 2024-04-22T22:28:25.961 INFO:teuthology.orchestra.run.smithi067.stdout:iscsi.foo.smithi067.rrokcu smithi067 running (3m) 3m ago 3m 69.4M - 3.5 e1d6a67b021e 88568d9a3e4b 2024-04-22T22:28:25.961 INFO:teuthology.orchestra.run.smithi067.stdout:mgr.x smithi086 *:8443 running (8m) 3m ago 8m 410M - 17.2.0 e1d6a67b021e cd2bbd4855c3 2024-04-22T22:28:25.961 INFO:teuthology.orchestra.run.smithi067.stdout:mgr.y smithi067 *:9283 running (10m) 3m ago 10m 453M - 17.2.0 e1d6a67b021e c5a816fd6dde 2024-04-22T22:28:25.961 INFO:teuthology.orchestra.run.smithi067.stdout:mon.a smithi067 running (10m) 3m ago 10m 47.5M 2048M 17.2.0 e1d6a67b021e fbf59dc0efb6 2024-04-22T22:28:25.961 INFO:teuthology.orchestra.run.smithi067.stdout:mon.b smithi086 running (8m) 3m ago 8m 40.7M 2048M 17.2.0 e1d6a67b021e be2e0891a577 2024-04-22T22:28:25.961 INFO:teuthology.orchestra.run.smithi067.stdout:mon.c smithi067 running (8m) 3m ago 8m 39.0M 2048M 17.2.0 e1d6a67b021e e93c724294c9 2024-04-22T22:28:25.961 INFO:teuthology.orchestra.run.smithi067.stdout:node-exporter.a smithi067 *:9100 running (4m) 3m ago 4m 15.4M - 1dbe0e931976 865795666bfc 2024-04-22T22:28:25.961 INFO:teuthology.orchestra.run.smithi067.stdout:node-exporter.b smithi086 *:9100 running (4m) 3m ago 4m 14.1M - 1dbe0e931976 bc12658d1c9c 2024-04-22T22:28:25.961 INFO:teuthology.orchestra.run.smithi067.stdout:osd.0 smithi067 running (8m) 3m ago 8m 48.7M 4016M 17.2.0 e1d6a67b021e 2774168bdf21 2024-04-22T22:28:25.961 INFO:teuthology.orchestra.run.smithi067.stdout:osd.1 smithi067 running (7m) 3m ago 7m 49.1M 4016M 17.2.0 e1d6a67b021e c07f4f56fb85 2024-04-22T22:28:25.961 INFO:teuthology.orchestra.run.smithi067.stdout:osd.2 smithi067 running (7m) 3m ago 7m 47.2M 4016M 17.2.0 e1d6a67b021e 5431738fc1b7 2024-04-22T22:28:25.961 INFO:teuthology.orchestra.run.smithi067.stdout:osd.3 smithi067 running (7m) 3m ago 7m 45.1M 4016M 17.2.0 e1d6a67b021e ea1ecfb9c931 2024-04-22T22:28:25.961 INFO:teuthology.orchestra.run.smithi067.stdout:osd.4 smithi086 running (6m) 3m ago 6m 45.6M 4272M 17.2.0 e1d6a67b021e 4196c45a856b 2024-04-22T22:28:25.961 INFO:teuthology.orchestra.run.smithi067.stdout:osd.5 smithi086 running (6m) 3m ago 6m 44.5M 4272M 17.2.0 e1d6a67b021e 6c685d1d93a3 2024-04-22T22:28:25.961 INFO:teuthology.orchestra.run.smithi067.stdout:osd.6 smithi086 running (6m) 3m ago 6m 44.1M 4272M 17.2.0 e1d6a67b021e 7e586ba77fac 2024-04-22T22:28:25.962 INFO:teuthology.orchestra.run.smithi067.stdout:osd.7 smithi086 running (5m) 3m ago 5m 45.3M 4272M 17.2.0 e1d6a67b021e fa3c88c96bae 2024-04-22T22:28:25.962 INFO:teuthology.orchestra.run.smithi067.stdout:prometheus.a smithi086 *:9095 running (3m) 3m ago 4m 46.0M - 514e6a882f6e 8531ba0d81dc 2024-04-22T22:28:25.962 INFO:teuthology.orchestra.run.smithi067.stdout:rgw.foo.smithi067.bekyzf smithi067 *:8000 running (3m) 3m ago 3m 70.5M - 17.2.0 e1d6a67b021e df1f503ccd1b 2024-04-22T22:28:25.962 INFO:teuthology.orchestra.run.smithi067.stdout:rgw.foo.smithi086.cofxku smithi086 *:8000 running (3m) 3m ago 3m 70.2M - 17.2.0 e1d6a67b021e 66e404643e32 2024-04-22T22:28:26.520 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -e sha1=430e09df97c8fc7dc2b2ae424f68ed11366c540f -- bash -c 'ceph versions' 2024-04-22T22:28:27.513 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:27 smithi067 ceph-mon[21875]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:28:27.513 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:27 smithi067 ceph-mon[21875]: from='client.14982 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:28:27.513 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:27 smithi067 ceph-mon[26633]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:28:27.513 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:27 smithi067 ceph-mon[26633]: from='client.14982 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:28:27.761 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:27 smithi086 ceph-mon[22689]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:28:27.761 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:27 smithi086 ceph-mon[22689]: from='client.14982 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:28:28.575 INFO:teuthology.orchestra.run.smithi067.stdout:{ 2024-04-22T22:28:28.575 INFO:teuthology.orchestra.run.smithi067.stdout: "mon": { 2024-04-22T22:28:28.575 INFO:teuthology.orchestra.run.smithi067.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-22T22:28:28.575 INFO:teuthology.orchestra.run.smithi067.stdout: }, 2024-04-22T22:28:28.576 INFO:teuthology.orchestra.run.smithi067.stdout: "mgr": { 2024-04-22T22:28:28.576 INFO:teuthology.orchestra.run.smithi067.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T22:28:28.576 INFO:teuthology.orchestra.run.smithi067.stdout: }, 2024-04-22T22:28:28.576 INFO:teuthology.orchestra.run.smithi067.stdout: "osd": { 2024-04-22T22:28:28.576 INFO:teuthology.orchestra.run.smithi067.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T22:28:28.576 INFO:teuthology.orchestra.run.smithi067.stdout: }, 2024-04-22T22:28:28.576 INFO:teuthology.orchestra.run.smithi067.stdout: "mds": {}, 2024-04-22T22:28:28.576 INFO:teuthology.orchestra.run.smithi067.stdout: "rgw": { 2024-04-22T22:28:28.576 INFO:teuthology.orchestra.run.smithi067.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T22:28:28.576 INFO:teuthology.orchestra.run.smithi067.stdout: }, 2024-04-22T22:28:28.576 INFO:teuthology.orchestra.run.smithi067.stdout: "overall": { 2024-04-22T22:28:28.576 INFO:teuthology.orchestra.run.smithi067.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 15 2024-04-22T22:28:28.576 INFO:teuthology.orchestra.run.smithi067.stdout: } 2024-04-22T22:28:28.576 INFO:teuthology.orchestra.run.smithi067.stdout:} 2024-04-22T22:28:29.244 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -e sha1=430e09df97c8fc7dc2b2ae424f68ed11366c540f -- bash -c 'ceph -s' 2024-04-22T22:28:29.513 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:29 smithi067 ceph-mon[21875]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:29.513 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:29 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/4181835869' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T22:28:29.513 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:29 smithi067 ceph-mon[26633]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:29.513 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:29 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/4181835869' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T22:28:29.761 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:29 smithi086 ceph-mon[22689]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:29.761 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:29 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/4181835869' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T22:28:30.513 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:30 smithi067 ceph-mon[21875]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:28:30.513 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:30 smithi067 ceph-mon[26633]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:28:30.761 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:30 smithi086 ceph-mon[22689]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:28:30.799 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:28:30 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:28:30] "GET /metrics HTTP/1.1" 200 207930 "" "Prometheus/2.33.4" 2024-04-22T22:28:31.293 INFO:teuthology.orchestra.run.smithi067.stdout: cluster: 2024-04-22T22:28:31.294 INFO:teuthology.orchestra.run.smithi067.stdout: id: 1ab1a3a8-00f6-11ef-bc93-c7b262605968 2024-04-22T22:28:31.294 INFO:teuthology.orchestra.run.smithi067.stdout: health: HEALTH_OK 2024-04-22T22:28:31.294 INFO:teuthology.orchestra.run.smithi067.stdout: 2024-04-22T22:28:31.294 INFO:teuthology.orchestra.run.smithi067.stdout: services: 2024-04-22T22:28:31.294 INFO:teuthology.orchestra.run.smithi067.stdout: mon: 3 daemons, quorum a,c,b (age 8m) 2024-04-22T22:28:31.294 INFO:teuthology.orchestra.run.smithi067.stdout: mgr: y(active, since 5m), standbys: x 2024-04-22T22:28:31.294 INFO:teuthology.orchestra.run.smithi067.stdout: osd: 8 osds: 8 up (since 5m), 8 in (since 5m) 2024-04-22T22:28:31.295 INFO:teuthology.orchestra.run.smithi067.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-04-22T22:28:31.295 INFO:teuthology.orchestra.run.smithi067.stdout: 2024-04-22T22:28:31.295 INFO:teuthology.orchestra.run.smithi067.stdout: data: 2024-04-22T22:28:31.295 INFO:teuthology.orchestra.run.smithi067.stdout: pools: 6 pools, 161 pgs 2024-04-22T22:28:31.295 INFO:teuthology.orchestra.run.smithi067.stdout: objects: 209 objects, 585 KiB 2024-04-22T22:28:31.295 INFO:teuthology.orchestra.run.smithi067.stdout: usage: 64 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:28:31.295 INFO:teuthology.orchestra.run.smithi067.stdout: pgs: 161 active+clean 2024-04-22T22:28:31.295 INFO:teuthology.orchestra.run.smithi067.stdout: 2024-04-22T22:28:31.295 INFO:teuthology.orchestra.run.smithi067.stdout: io: 2024-04-22T22:28:31.295 INFO:teuthology.orchestra.run.smithi067.stdout: client: 1.2 KiB/s rd, 1 op/s rd, 0 op/s wr 2024-04-22T22:28:31.296 INFO:teuthology.orchestra.run.smithi067.stdout: 2024-04-22T22:28:31.672 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:31 smithi067 ceph-mon[21875]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:28:31.672 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:31 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/3163083544' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-22T22:28:31.672 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:31 smithi067 ceph-mon[26633]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:28:31.672 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:31 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/3163083544' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-22T22:28:31.761 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:31 smithi086 ceph-mon[22689]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:28:31.762 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:31 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/3163083544' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-22T22:28:31.762 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:28:31 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:28:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:28:32.081 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -e sha1=430e09df97c8fc7dc2b2ae424f68ed11366c540f -- bash -c 'ceph orch ls' 2024-04-22T22:28:33.644 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:33 smithi067 ceph-mon[21875]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:33.644 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:33 smithi067 ceph-mon[26633]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:33.761 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:33 smithi086 ceph-mon[22689]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:34.034 INFO:teuthology.orchestra.run.smithi067.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-04-22T22:28:34.035 INFO:teuthology.orchestra.run.smithi067.stdout:alertmanager ?:9093,9094 1/1 3m ago 5m smithi067=a;count:1 2024-04-22T22:28:34.035 INFO:teuthology.orchestra.run.smithi067.stdout:grafana ?:3000 1/1 3m ago 5m smithi086=a;count:1 2024-04-22T22:28:34.035 INFO:teuthology.orchestra.run.smithi067.stdout:iscsi.foo 1/1 3m ago 3m count:1 2024-04-22T22:28:34.035 INFO:teuthology.orchestra.run.smithi067.stdout:mgr 2/2 3m ago 8m smithi067=y;smithi086=x;count:2 2024-04-22T22:28:34.035 INFO:teuthology.orchestra.run.smithi067.stdout:mon 3/3 3m ago 9m smithi067:172.21.15.67=a;smithi067:[v2:172.21.15.67:3301,v1:172.21.15.67:6790]=c;smithi086:172.21.15.86=b;count:3 2024-04-22T22:28:34.035 INFO:teuthology.orchestra.run.smithi067.stdout:node-exporter ?:9100 2/2 3m ago 5m smithi067=a;smithi086=b;count:2 2024-04-22T22:28:34.035 INFO:teuthology.orchestra.run.smithi067.stdout:osd 8 3m ago - 2024-04-22T22:28:34.035 INFO:teuthology.orchestra.run.smithi067.stdout:prometheus ?:9095 1/1 3m ago 5m smithi086=a;count:1 2024-04-22T22:28:34.035 INFO:teuthology.orchestra.run.smithi067.stdout:rgw.foo ?:8000 2/2 3m ago 3m count:2 2024-04-22T22:28:34.638 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -e sha1=430e09df97c8fc7dc2b2ae424f68ed11366c540f -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-22T22:28:35.513 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:35 smithi067 ceph-mon[21875]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:35.513 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:35 smithi067 ceph-mon[21875]: from='client.15000 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:28:35.513 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:35 smithi067 ceph-mon[26633]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:35.513 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:35 smithi067 ceph-mon[26633]: from='client.15000 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:28:35.761 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:35 smithi086 ceph-mon[22689]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:35.762 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:35 smithi086 ceph-mon[22689]: from='client.15000 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:28:37.106 INFO:teuthology.orchestra.run.smithi067.stdout:Scheduled to redeploy mgr.x on host 'smithi086' 2024-04-22T22:28:37.417 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:37 smithi067 ceph-mon[21875]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:28:37.417 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:37 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/2933582478' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-22T22:28:37.418 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:37 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:28:37.418 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:37 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:28:37.418 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:37 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:28:37.418 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:37 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:28:37.418 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:37 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T22:28:37.418 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:37 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T22:28:37.418 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:37 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:28:37.418 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:37 smithi067 ceph-mon[26633]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:28:37.418 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:37 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/2933582478' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-22T22:28:37.418 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:37 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:28:37.418 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:37 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:28:37.418 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:37 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:28:37.419 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:37 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:28:37.419 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:37 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T22:28:37.419 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:37 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T22:28:37.419 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:37 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:28:37.512 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:37 smithi086 ceph-mon[22689]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:28:37.512 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:37 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/2933582478' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-22T22:28:37.512 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:37 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:28:37.512 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:37 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:28:37.512 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:37 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:28:37.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:37 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:28:37.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:37 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T22:28:37.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:37 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T22:28:37.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:37 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:28:37.857 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -e sha1=430e09df97c8fc7dc2b2ae424f68ed11366c540f -- bash -c 'ceph orch ps --refresh' 2024-04-22T22:28:38.435 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:38 smithi067 ceph-mon[21875]: from='client.24913 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:430e09df97c8fc7dc2b2ae424f68ed11366c540f", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:28:38.435 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:38 smithi067 ceph-mon[21875]: Schedule redeploy daemon mgr.x 2024-04-22T22:28:38.435 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:38 smithi067 ceph-mon[21875]: Deploying daemon mgr.x on smithi086 2024-04-22T22:28:38.435 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:38 smithi067 ceph-mon[26633]: from='client.24913 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:430e09df97c8fc7dc2b2ae424f68ed11366c540f", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:28:38.436 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:38 smithi067 ceph-mon[26633]: Schedule redeploy daemon mgr.x 2024-04-22T22:28:38.436 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:38 smithi067 ceph-mon[26633]: Deploying daemon mgr.x on smithi086 2024-04-22T22:28:38.762 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:38 smithi086 ceph-mon[22689]: from='client.24913 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:430e09df97c8fc7dc2b2ae424f68ed11366c540f", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:28:38.762 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:38 smithi086 ceph-mon[22689]: Schedule redeploy daemon mgr.x 2024-04-22T22:28:38.762 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:38 smithi086 ceph-mon[22689]: Deploying daemon mgr.x on smithi086 2024-04-22T22:28:39.589 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:39 smithi067 ceph-mon[21875]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:39.590 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:39 smithi067 ceph-mon[26633]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:39.762 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:39 smithi086 ceph-mon[22689]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:40.177 INFO:teuthology.orchestra.run.smithi067.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T22:28:40.177 INFO:teuthology.orchestra.run.smithi067.stdout:alertmanager.a smithi067 *:9093,9094 running (3m) 3m ago 4m 17.6M - ba2b418f427c 9d0e1f18c1cb 2024-04-22T22:28:40.177 INFO:teuthology.orchestra.run.smithi067.stdout:grafana.a smithi086 *:3000 running (4m) 3m ago 4m 47.6M - 8.3.5 dad864ee21e9 5d6c83f8480c 2024-04-22T22:28:40.177 INFO:teuthology.orchestra.run.smithi067.stdout:iscsi.foo.smithi067.rrokcu smithi067 running (3m) 3m ago 3m 69.4M - 3.5 e1d6a67b021e 88568d9a3e4b 2024-04-22T22:28:40.177 INFO:teuthology.orchestra.run.smithi067.stdout:mgr.x smithi086 *:8443 running (8m) 3m ago 8m 410M - 17.2.0 e1d6a67b021e cd2bbd4855c3 2024-04-22T22:28:40.177 INFO:teuthology.orchestra.run.smithi067.stdout:mgr.y smithi067 *:9283 running (10m) 3m ago 10m 453M - 17.2.0 e1d6a67b021e c5a816fd6dde 2024-04-22T22:28:40.177 INFO:teuthology.orchestra.run.smithi067.stdout:mon.a smithi067 running (10m) 3m ago 10m 47.5M 2048M 17.2.0 e1d6a67b021e fbf59dc0efb6 2024-04-22T22:28:40.177 INFO:teuthology.orchestra.run.smithi067.stdout:mon.b smithi086 running (9m) 3m ago 9m 40.7M 2048M 17.2.0 e1d6a67b021e be2e0891a577 2024-04-22T22:28:40.177 INFO:teuthology.orchestra.run.smithi067.stdout:mon.c smithi067 running (9m) 3m ago 9m 39.0M 2048M 17.2.0 e1d6a67b021e e93c724294c9 2024-04-22T22:28:40.177 INFO:teuthology.orchestra.run.smithi067.stdout:node-exporter.a smithi067 *:9100 running (5m) 3m ago 5m 15.4M - 1dbe0e931976 865795666bfc 2024-04-22T22:28:40.177 INFO:teuthology.orchestra.run.smithi067.stdout:node-exporter.b smithi086 *:9100 running (5m) 3m ago 5m 14.1M - 1dbe0e931976 bc12658d1c9c 2024-04-22T22:28:40.177 INFO:teuthology.orchestra.run.smithi067.stdout:osd.0 smithi067 running (8m) 3m ago 8m 48.7M 4016M 17.2.0 e1d6a67b021e 2774168bdf21 2024-04-22T22:28:40.178 INFO:teuthology.orchestra.run.smithi067.stdout:osd.1 smithi067 running (8m) 3m ago 8m 49.1M 4016M 17.2.0 e1d6a67b021e c07f4f56fb85 2024-04-22T22:28:40.178 INFO:teuthology.orchestra.run.smithi067.stdout:osd.2 smithi067 running (7m) 3m ago 7m 47.2M 4016M 17.2.0 e1d6a67b021e 5431738fc1b7 2024-04-22T22:28:40.178 INFO:teuthology.orchestra.run.smithi067.stdout:osd.3 smithi067 running (7m) 3m ago 7m 45.1M 4016M 17.2.0 e1d6a67b021e ea1ecfb9c931 2024-04-22T22:28:40.178 INFO:teuthology.orchestra.run.smithi067.stdout:osd.4 smithi086 running (7m) 3m ago 7m 45.6M 4272M 17.2.0 e1d6a67b021e 4196c45a856b 2024-04-22T22:28:40.178 INFO:teuthology.orchestra.run.smithi067.stdout:osd.5 smithi086 running (6m) 3m ago 6m 44.5M 4272M 17.2.0 e1d6a67b021e 6c685d1d93a3 2024-04-22T22:28:40.178 INFO:teuthology.orchestra.run.smithi067.stdout:osd.6 smithi086 running (6m) 3m ago 6m 44.1M 4272M 17.2.0 e1d6a67b021e 7e586ba77fac 2024-04-22T22:28:40.178 INFO:teuthology.orchestra.run.smithi067.stdout:osd.7 smithi086 running (5m) 3m ago 5m 45.3M 4272M 17.2.0 e1d6a67b021e fa3c88c96bae 2024-04-22T22:28:40.178 INFO:teuthology.orchestra.run.smithi067.stdout:prometheus.a smithi086 *:9095 running (3m) 3m ago 5m 46.0M - 514e6a882f6e 8531ba0d81dc 2024-04-22T22:28:40.178 INFO:teuthology.orchestra.run.smithi067.stdout:rgw.foo.smithi067.bekyzf smithi067 *:8000 running (3m) 3m ago 3m 70.5M - 17.2.0 e1d6a67b021e df1f503ccd1b 2024-04-22T22:28:40.178 INFO:teuthology.orchestra.run.smithi067.stdout:rgw.foo.smithi086.cofxku smithi086 *:8000 running (3m) 3m ago 3m 70.2M - 17.2.0 e1d6a67b021e 66e404643e32 2024-04-22T22:28:40.491 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:40 smithi067 ceph-mon[21875]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:28:40.491 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:40 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:28:40.491 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:40 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:28:40.491 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:40 smithi067 ceph-mon[26633]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:28:40.492 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:40 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:28:40.492 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:40 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:28:40.511 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:40 smithi086 ceph-mon[22689]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:28:40.512 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:40 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:28:40.512 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:40 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:28:40.755 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:28:40 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:28:40] "GET /metrics HTTP/1.1" 200 207930 "" "Prometheus/2.33.4" 2024-04-22T22:28:40.756 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -e sha1=430e09df97c8fc7dc2b2ae424f68ed11366c540f -- bash -c 'sleep 180' 2024-04-22T22:28:41.513 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:41 smithi067 ceph-mon[21875]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:28:41.513 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:41 smithi067 ceph-mon[21875]: from='client.24919 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:28:41.513 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:41 smithi067 ceph-mon[26633]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:28:41.513 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:41 smithi067 ceph-mon[26633]: from='client.24919 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:28:41.762 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:41 smithi086 ceph-mon[22689]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:28:41.762 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:41 smithi086 ceph-mon[22689]: from='client.24919 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:28:41.762 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:28:41 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:28:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:28:43.512 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:43 smithi086 ceph-mon[22689]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:43.703 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:43 smithi067 ceph-mon[21875]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:43.704 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:43 smithi067 ceph-mon[26633]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:44.966 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:44 smithi067 ceph-mon[21875]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:44.966 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:44 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:28:44.966 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:44 smithi067 ceph-mon[26633]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:44.966 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:44 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:28:45.012 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:44 smithi086 ceph-mon[22689]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:45.012 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:44 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:28:46.762 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:46 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:28:46.762 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:46 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:28:46.762 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:46 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T22:28:46.762 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:46 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T22:28:46.762 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:46 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:28:46.762 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:46 smithi086 ceph-mon[22689]: Deploying daemon mgr.x on smithi086 2024-04-22T22:28:46.762 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:46 smithi086 ceph-mon[22689]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:28:46.763 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:46 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:28:46.763 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:46 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:28:46.763 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:46 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T22:28:46.763 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:46 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T22:28:46.763 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:46 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:28:46.763 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:46 smithi067 ceph-mon[26633]: Deploying daemon mgr.x on smithi086 2024-04-22T22:28:46.763 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:46 smithi067 ceph-mon[26633]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:28:46.763 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:46 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:28:46.763 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:46 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:28:46.764 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:46 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T22:28:46.764 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:46 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T22:28:46.764 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:46 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:28:46.764 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:46 smithi067 ceph-mon[21875]: Deploying daemon mgr.x on smithi086 2024-04-22T22:28:46.764 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:46 smithi067 ceph-mon[21875]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:28:48.755 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:48 smithi067 ceph-mon[21875]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:48.755 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:48 smithi067 ceph-mon[26633]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:48.762 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:48 smithi086 ceph-mon[22689]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:49.762 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:49 smithi086 ceph-mon[22689]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:28:49.763 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:49 smithi067 ceph-mon[21875]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:28:49.763 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:49 smithi067 ceph-mon[26633]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:28:51.012 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:50 smithi086 ceph-mon[22689]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:28:51.013 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:50 smithi067 ceph-mon[21875]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:28:51.013 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:50 smithi067 ceph-mon[26633]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:28:51.013 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:28:50 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:28:50] "GET /metrics HTTP/1.1" 200 207923 "" "Prometheus/2.33.4" 2024-04-22T22:28:51.671 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:28:51 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:28:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:28:53.012 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:52 smithi086 ceph-mon[22689]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:53.013 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:52 smithi067 ceph-mon[21875]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:53.013 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:52 smithi067 ceph-mon[26633]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:55.012 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:54 smithi086 ceph-mon[22689]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:55.013 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:54 smithi067 ceph-mon[21875]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:55.013 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:54 smithi067 ceph-mon[26633]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:57.012 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:56 smithi086 ceph-mon[22689]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:28:57.013 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:56 smithi067 ceph-mon[21875]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:28:57.013 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:56 smithi067 ceph-mon[26633]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:28:59.012 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:58 smithi086 ceph-mon[22689]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:59.013 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:58 smithi067 ceph-mon[21875]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:28:59.013 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:58 smithi067 ceph-mon[26633]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:00.012 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:28:59 smithi086 ceph-mon[22689]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:29:00.013 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:28:59 smithi067 ceph-mon[21875]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:29:00.013 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:28:59 smithi067 ceph-mon[26633]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:29:01.012 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:00 smithi086 ceph-mon[22689]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:29:01.013 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:00 smithi067 ceph-mon[21875]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:29:01.013 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:00 smithi067 ceph-mon[26633]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:29:01.014 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:29:00 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:29:00] "GET /metrics HTTP/1.1" 200 207941 "" "Prometheus/2.33.4" 2024-04-22T22:29:01.762 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:29:01 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:29:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:29:03.012 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:02 smithi086 ceph-mon[22689]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:03.013 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:02 smithi067 ceph-mon[21875]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:03.014 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:02 smithi067 ceph-mon[26633]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:05.012 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:04 smithi086 ceph-mon[22689]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:05.013 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:04 smithi067 ceph-mon[21875]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:05.013 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:04 smithi067 ceph-mon[26633]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:07.012 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:06 smithi086 ceph-mon[22689]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:29:07.013 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:06 smithi067 ceph-mon[21875]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:29:07.013 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:06 smithi067 ceph-mon[26633]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:29:09.012 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:08 smithi086 ceph-mon[22689]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:09.013 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:08 smithi067 ceph-mon[21875]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:09.013 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:08 smithi067 ceph-mon[26633]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:10.262 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:09 smithi086 ceph-mon[22689]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:29:10.263 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:09 smithi067 ceph-mon[21875]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:29:10.263 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:09 smithi067 ceph-mon[26633]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:29:11.013 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:10 smithi067 ceph-mon[21875]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:29:11.014 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:10 smithi067 ceph-mon[26633]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:29:11.014 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:29:10 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:29:10] "GET /metrics HTTP/1.1" 200 207941 "" "Prometheus/2.33.4" 2024-04-22T22:29:11.163 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:10 smithi086 ceph-mon[22689]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:29:11.762 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:29:11 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:29:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:29:13.262 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:12 smithi086 ceph-mon[22689]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:13.263 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:12 smithi067 ceph-mon[21875]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:13.263 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:12 smithi067 ceph-mon[26633]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:14.262 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:13 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:29:14.262 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:13 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:29:14.263 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:13 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:29:14.263 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:13 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:29:14.263 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:13 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:29:14.264 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:13 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:29:15.262 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:14 smithi086 ceph-mon[22689]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:15.263 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:14 smithi067 ceph-mon[21875]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:15.263 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:14 smithi067 ceph-mon[26633]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:17.262 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:16 smithi086 ceph-mon[22689]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:29:17.263 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:16 smithi067 ceph-mon[21875]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:29:17.263 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:16 smithi067 ceph-mon[26633]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:29:19.262 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:18 smithi086 ceph-mon[22689]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:19.263 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:18 smithi067 ceph-mon[21875]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:19.263 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:18 smithi067 ceph-mon[26633]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:20.262 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:19 smithi086 ceph-mon[22689]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:29:20.263 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:19 smithi067 ceph-mon[21875]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:29:20.264 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:19 smithi067 ceph-mon[26633]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:29:21.014 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:20 smithi067 ceph-mon[21875]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:29:21.014 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:20 smithi067 ceph-mon[26633]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:29:21.015 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:29:20 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:29:20] "GET /metrics HTTP/1.1" 200 207954 "" "Prometheus/2.33.4" 2024-04-22T22:29:21.262 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:20 smithi086 ceph-mon[22689]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:29:21.762 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:29:21 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:29:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:29:23.211 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:22 smithi067 ceph-mon[21875]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:23.211 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:22 smithi067 ceph-mon[26633]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:23.262 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:22 smithi086 ceph-mon[22689]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:25.262 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:24 smithi086 ceph-mon[22689]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:25.263 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:24 smithi067 ceph-mon[21875]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:25.263 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:24 smithi067 ceph-mon[26633]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:27.262 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:26 smithi086 ceph-mon[22689]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:29:27.263 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:26 smithi067 ceph-mon[21875]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:29:27.263 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:26 smithi067 ceph-mon[26633]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:29:29.262 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:28 smithi086 ceph-mon[22689]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:29.263 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:28 smithi067 ceph-mon[21875]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:29.264 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:28 smithi067 ceph-mon[26633]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:30.262 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:29 smithi086 ceph-mon[22689]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:29:30.264 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:29 smithi067 ceph-mon[26633]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:29:30.264 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:29 smithi067 ceph-mon[21875]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:29:31.014 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:30 smithi067 ceph-mon[21875]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:29:31.014 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:30 smithi067 ceph-mon[26633]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:29:31.014 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:29:30 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:29:30] "GET /metrics HTTP/1.1" 200 207944 "" "Prometheus/2.33.4" 2024-04-22T22:29:31.263 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:30 smithi086 ceph-mon[22689]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:29:31.762 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:29:31 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:29:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:29:33.014 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:32 smithi067 ceph-mon[21875]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:33.014 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:32 smithi067 ceph-mon[26633]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:33.262 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:32 smithi086 ceph-mon[22689]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:35.262 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:34 smithi086 ceph-mon[22689]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:35.263 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:34 smithi067 ceph-mon[21875]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:35.264 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:34 smithi067 ceph-mon[26633]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:37.262 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:36 smithi086 ceph-mon[22689]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:29:37.264 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:36 smithi067 ceph-mon[21875]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:29:37.264 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:36 smithi067 ceph-mon[26633]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:29:39.262 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:38 smithi086 ceph-mon[22689]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:39.264 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:38 smithi067 ceph-mon[21875]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:39.264 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:38 smithi067 ceph-mon[26633]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:40.263 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:39 smithi086 ceph-mon[22689]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:29:40.264 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:39 smithi067 ceph-mon[21875]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:29:40.264 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:39 smithi067 ceph-mon[26633]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:29:40.837 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:29:40 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:29:40] "GET /metrics HTTP/1.1" 200 207944 "" "Prometheus/2.33.4" 2024-04-22T22:29:41.164 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:40 smithi086 ceph-mon[22689]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:29:41.264 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:40 smithi067 ceph-mon[21875]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:29:41.264 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:40 smithi067 ceph-mon[26633]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:29:41.763 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:29:41 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:29:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:29:43.263 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:42 smithi086 ceph-mon[22689]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:43.264 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:42 smithi067 ceph-mon[21875]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:43.264 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:42 smithi067 ceph-mon[26633]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:45.263 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:44 smithi086 ceph-mon[22689]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:45.264 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:44 smithi067 ceph-mon[21875]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:45.264 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:44 smithi067 ceph-mon[26633]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:46.102 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:45 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:29:46.102 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:45 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:29:46.122 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:45 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:29:46.122 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:45 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:29:46.122 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:45 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:29:46.122 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:45 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:29:47.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:47 smithi086 ceph-mon[22689]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:29:47.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:47 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:29:47.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:47 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:29:47.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:47 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:29:47.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:47 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T22:29:47.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:47 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T22:29:47.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:47 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:29:47.514 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:47 smithi067 ceph-mon[21875]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:29:47.514 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:47 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:29:47.514 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:47 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:29:47.514 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:47 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:29:47.514 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:47 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T22:29:47.514 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:47 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T22:29:47.514 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:47 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:29:47.515 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:47 smithi067 ceph-mon[26633]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:29:47.515 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:47 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:29:47.515 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:47 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:29:47.515 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:47 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:29:47.515 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:47 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T22:29:47.515 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:47 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T22:29:47.515 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:47 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:29:48.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:48 smithi086 ceph-mon[22689]: Deploying daemon mgr.x on smithi086 2024-04-22T22:29:48.514 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:48 smithi067 ceph-mon[21875]: Deploying daemon mgr.x on smithi086 2024-04-22T22:29:48.514 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:48 smithi067 ceph-mon[26633]: Deploying daemon mgr.x on smithi086 2024-04-22T22:29:49.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:49 smithi086 ceph-mon[22689]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:49.514 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:49 smithi067 ceph-mon[21875]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:49.514 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:49 smithi067 ceph-mon[26633]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:50.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:50 smithi086 ceph-mon[22689]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:29:50.514 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:50 smithi067 ceph-mon[21875]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:29:50.514 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:50 smithi067 ceph-mon[26633]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:29:51.014 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:29:50 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:29:50] "GET /metrics HTTP/1.1" 200 207951 "" "Prometheus/2.33.4" 2024-04-22T22:29:51.418 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:51 smithi086 ceph-mon[22689]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:29:51.514 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:51 smithi067 ceph-mon[21875]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:29:51.514 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:51 smithi067 ceph-mon[26633]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:29:51.763 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:29:51 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:29:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:29:53.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:53 smithi086 ceph-mon[22689]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:53.515 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:53 smithi067 ceph-mon[21875]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:53.515 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:53 smithi067 ceph-mon[26633]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:55.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:55 smithi086 ceph-mon[22689]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:55.514 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:55 smithi067 ceph-mon[21875]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:55.514 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:55 smithi067 ceph-mon[26633]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:57.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:57 smithi086 ceph-mon[22689]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:29:57.514 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:57 smithi067 ceph-mon[21875]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:29:57.515 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:57 smithi067 ceph-mon[26633]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:29:59.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:29:59 smithi086 ceph-mon[22689]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:59.514 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:29:59 smithi067 ceph-mon[21875]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:29:59.514 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:29:59 smithi067 ceph-mon[26633]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:00.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:00 smithi086 ceph-mon[22689]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:30:00.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:00 smithi086 ceph-mon[22689]: overall HEALTH_OK 2024-04-22T22:30:00.514 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:00 smithi067 ceph-mon[21875]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:30:00.514 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:00 smithi067 ceph-mon[21875]: overall HEALTH_OK 2024-04-22T22:30:00.514 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:00 smithi067 ceph-mon[26633]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:30:00.514 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:00 smithi067 ceph-mon[26633]: overall HEALTH_OK 2024-04-22T22:30:01.014 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:30:00 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:30:00] "GET /metrics HTTP/1.1" 200 207952 "" "Prometheus/2.33.4" 2024-04-22T22:30:01.417 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:01 smithi086 ceph-mon[22689]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:30:01.514 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:01 smithi067 ceph-mon[26633]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:30:01.514 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:01 smithi067 ceph-mon[21875]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:30:01.763 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:30:01 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:30:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:30:03.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:03 smithi086 ceph-mon[22689]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:03.514 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:03 smithi067 ceph-mon[26633]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:03.514 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:03 smithi067 ceph-mon[21875]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:05.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:05 smithi086 ceph-mon[22689]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:05.514 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:05 smithi067 ceph-mon[26633]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:05.514 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:05 smithi067 ceph-mon[21875]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:07.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:07 smithi086 ceph-mon[22689]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:30:07.514 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:07 smithi067 ceph-mon[26633]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:30:07.514 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:07 smithi067 ceph-mon[21875]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:30:09.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:09 smithi086 ceph-mon[22689]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:09.514 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:09 smithi067 ceph-mon[21875]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:09.514 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:09 smithi067 ceph-mon[26633]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:10.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:10 smithi086 ceph-mon[22689]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:30:10.514 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:10 smithi067 ceph-mon[21875]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:30:10.514 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:10 smithi067 ceph-mon[26633]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:30:11.014 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:30:10 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:30:10] "GET /metrics HTTP/1.1" 200 207952 "" "Prometheus/2.33.4" 2024-04-22T22:30:11.418 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:11 smithi086 ceph-mon[22689]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:30:11.516 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:11 smithi067 ceph-mon[21875]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:30:11.516 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:11 smithi067 ceph-mon[26633]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:30:11.763 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:30:11 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:30:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:30:13.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:13 smithi086 ceph-mon[22689]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:13.514 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:13 smithi067 ceph-mon[21875]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:13.515 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:13 smithi067 ceph-mon[26633]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:14.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:14 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:30:14.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:14 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:30:14.514 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:14 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:30:14.515 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:14 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:30:14.515 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:14 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:30:14.515 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:14 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:30:15.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:15 smithi086 ceph-mon[22689]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:15.514 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:15 smithi067 ceph-mon[21875]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:15.514 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:15 smithi067 ceph-mon[26633]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:17.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:17 smithi086 ceph-mon[22689]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:30:17.514 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:17 smithi067 ceph-mon[21875]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:30:17.514 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:17 smithi067 ceph-mon[26633]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:30:19.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:19 smithi086 ceph-mon[22689]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:19.514 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:19 smithi067 ceph-mon[21875]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:19.515 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:19 smithi067 ceph-mon[26633]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:20.514 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:20 smithi086 ceph-mon[22689]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:30:20.515 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:20 smithi067 ceph-mon[21875]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:30:20.515 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:20 smithi067 ceph-mon[26633]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:30:21.015 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:30:20 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:30:20] "GET /metrics HTTP/1.1" 200 207949 "" "Prometheus/2.33.4" 2024-04-22T22:30:21.514 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:21 smithi086 ceph-mon[22689]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:30:21.514 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:30:21 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:30:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:30:21.514 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:21 smithi067 ceph-mon[21875]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:30:21.515 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:21 smithi067 ceph-mon[26633]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:30:23.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:23 smithi086 ceph-mon[22689]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:23.514 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:23 smithi067 ceph-mon[21875]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:23.514 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:23 smithi067 ceph-mon[26633]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:25.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:25 smithi086 ceph-mon[22689]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:25.514 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:25 smithi067 ceph-mon[21875]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:25.515 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:25 smithi067 ceph-mon[26633]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:27.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:27 smithi086 ceph-mon[22689]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:30:27.514 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:27 smithi067 ceph-mon[21875]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:30:27.514 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:27 smithi067 ceph-mon[26633]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:30:29.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:29 smithi086 ceph-mon[22689]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:29.514 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:29 smithi067 ceph-mon[21875]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:29.515 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:29 smithi067 ceph-mon[26633]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:30.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:30 smithi086 ceph-mon[22689]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:30:30.514 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:30 smithi067 ceph-mon[21875]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:30:30.515 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:30 smithi067 ceph-mon[26633]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:30:31.015 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:30:30 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:30:30] "GET /metrics HTTP/1.1" 200 207947 "" "Prometheus/2.33.4" 2024-04-22T22:30:31.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:31 smithi086 ceph-mon[22689]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:30:31.514 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:30:31 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:30:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:30:31.514 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:31 smithi067 ceph-mon[21875]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:30:31.515 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:31 smithi067 ceph-mon[26633]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:30:33.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:33 smithi086 ceph-mon[22689]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:33.514 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:33 smithi067 ceph-mon[21875]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:33.515 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:33 smithi067 ceph-mon[26633]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:35.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:35 smithi086 ceph-mon[22689]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:35.514 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:35 smithi067 ceph-mon[21875]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:35.515 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:35 smithi067 ceph-mon[26633]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:37.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:37 smithi086 ceph-mon[22689]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:30:37.514 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:37 smithi067 ceph-mon[26633]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:30:37.515 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:37 smithi067 ceph-mon[21875]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:30:39.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:39 smithi086 ceph-mon[22689]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:39.515 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:39 smithi067 ceph-mon[21875]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:39.515 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:39 smithi067 ceph-mon[26633]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:40.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:40 smithi086 ceph-mon[22689]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:30:40.516 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:40 smithi067 ceph-mon[21875]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:30:40.516 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:40 smithi067 ceph-mon[26633]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:30:41.015 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:30:40 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:30:40] "GET /metrics HTTP/1.1" 200 207947 "" "Prometheus/2.33.4" 2024-04-22T22:30:41.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:41 smithi086 ceph-mon[22689]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:30:41.514 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:30:41 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:30:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:30:41.514 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:41 smithi067 ceph-mon[21875]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:30:41.515 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:41 smithi067 ceph-mon[26633]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:30:43.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:43 smithi086 ceph-mon[22689]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:43.515 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:43 smithi067 ceph-mon[21875]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:43.515 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:43 smithi067 ceph-mon[26633]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:45.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:45 smithi086 ceph-mon[22689]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:45.515 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:45 smithi067 ceph-mon[21875]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:45.515 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:45 smithi067 ceph-mon[26633]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:47.514 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:47 smithi086 ceph-mon[22689]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:30:47.514 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:47 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:30:47.514 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:47 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:30:47.514 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:47 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:30:47.514 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:47 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T22:30:47.514 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:47 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T22:30:47.514 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:47 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:30:47.515 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:47 smithi067 ceph-mon[21875]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:30:47.515 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:47 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:30:47.515 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:47 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:30:47.515 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:47 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:30:47.515 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:47 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T22:30:47.515 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:47 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T22:30:47.515 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:47 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:30:47.515 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:47 smithi067 ceph-mon[26633]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:30:47.515 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:47 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:30:47.516 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:47 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:30:47.516 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:47 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:30:47.516 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:47 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T22:30:47.516 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:47 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T22:30:47.516 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:47 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:30:48.513 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:48 smithi086 ceph-mon[22689]: Deploying daemon mgr.x on smithi086 2024-04-22T22:30:48.514 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:48 smithi067 ceph-mon[21875]: Deploying daemon mgr.x on smithi086 2024-04-22T22:30:48.515 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:48 smithi067 ceph-mon[26633]: Deploying daemon mgr.x on smithi086 2024-04-22T22:30:49.514 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:49 smithi086 ceph-mon[22689]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:49.515 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:49 smithi067 ceph-mon[21875]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:49.515 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:49 smithi067 ceph-mon[26633]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:50.514 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:50 smithi086 ceph-mon[22689]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:30:50.515 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:50 smithi067 ceph-mon[21875]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:30:50.515 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:50 smithi067 ceph-mon[26633]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:30:51.015 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:30:50 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:30:50] "GET /metrics HTTP/1.1" 200 207944 "" "Prometheus/2.33.4" 2024-04-22T22:30:51.274 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:51 smithi067 ceph-mon[21875]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:30:51.275 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:51 smithi067 ceph-mon[26633]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:30:51.514 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:51 smithi086 ceph-mon[22689]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:30:51.514 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:30:51 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:30:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:30:53.514 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:53 smithi086 ceph-mon[22689]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:53.515 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:53 smithi067 ceph-mon[21875]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:53.515 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:53 smithi067 ceph-mon[26633]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:55.515 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:55 smithi067 ceph-mon[21875]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:55.515 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:55 smithi067 ceph-mon[26633]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:55.764 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:55 smithi086 ceph-mon[22689]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:57.764 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:57 smithi086 ceph-mon[22689]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:30:57.765 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:57 smithi067 ceph-mon[21875]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:30:57.766 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:57 smithi067 ceph-mon[26633]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:30:59.764 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:30:59 smithi086 ceph-mon[22689]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:59.765 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:30:59 smithi067 ceph-mon[21875]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:30:59.765 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:30:59 smithi067 ceph-mon[26633]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:00.576 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:00 smithi067 ceph-mon[21875]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:31:00.577 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:00 smithi067 ceph-mon[26633]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:31:00.764 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:00 smithi086 ceph-mon[22689]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:31:01.015 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:31:00 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:31:00] "GET /metrics HTTP/1.1" 200 207949 "" "Prometheus/2.33.4" 2024-04-22T22:31:01.694 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:01 smithi067 ceph-mon[21875]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:31:01.694 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:01 smithi067 ceph-mon[26633]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:31:01.764 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:01 smithi086 ceph-mon[22689]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:31:01.764 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:31:01 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:31:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:31:03.764 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:03 smithi086 ceph-mon[22689]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:03.765 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:03 smithi067 ceph-mon[21875]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:03.765 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:03 smithi067 ceph-mon[26633]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:05.764 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:05 smithi086 ceph-mon[22689]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:05.765 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:05 smithi067 ceph-mon[21875]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:05.765 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:05 smithi067 ceph-mon[26633]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:07.764 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:07 smithi086 ceph-mon[22689]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:31:07.765 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:07 smithi067 ceph-mon[21875]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:31:07.765 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:07 smithi067 ceph-mon[26633]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:31:09.764 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:09 smithi086 ceph-mon[22689]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:09.765 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:09 smithi067 ceph-mon[21875]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:09.765 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:09 smithi067 ceph-mon[26633]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:10.576 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:10 smithi067 ceph-mon[21875]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:31:10.577 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:10 smithi067 ceph-mon[26633]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:31:10.764 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:10 smithi086 ceph-mon[22689]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:31:11.015 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:31:10 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:31:10] "GET /metrics HTTP/1.1" 200 207949 "" "Prometheus/2.33.4" 2024-04-22T22:31:11.665 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:11 smithi086 ceph-mon[22689]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:31:11.666 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:31:11 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:31:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:31:11.765 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:11 smithi067 ceph-mon[21875]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:31:11.766 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:11 smithi067 ceph-mon[26633]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:31:13.764 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:13 smithi086 ceph-mon[22689]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:13.765 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:13 smithi067 ceph-mon[21875]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:13.765 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:13 smithi067 ceph-mon[26633]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:14.764 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:14 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:31:14.764 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:14 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:31:14.765 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:14 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:31:14.765 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:14 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:31:14.765 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:14 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-22T22:31:14.765 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:14 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-22T22:31:15.764 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:15 smithi086 ceph-mon[22689]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:15.765 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:15 smithi067 ceph-mon[21875]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:15.765 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:15 smithi067 ceph-mon[26633]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:17.764 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:17 smithi086 ceph-mon[22689]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:31:17.765 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:17 smithi067 ceph-mon[21875]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:31:17.765 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:17 smithi067 ceph-mon[26633]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:31:19.764 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:19 smithi086 ceph-mon[22689]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:19.765 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:19 smithi067 ceph-mon[21875]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:19.765 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:19 smithi067 ceph-mon[26633]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:20.578 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:20 smithi067 ceph-mon[21875]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:31:20.578 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:20 smithi067 ceph-mon[26633]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:31:20.764 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:20 smithi086 ceph-mon[22689]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:31:21.016 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:31:20 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:31:20] "GET /metrics HTTP/1.1" 200 207954 "" "Prometheus/2.33.4" 2024-04-22T22:31:21.764 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:21 smithi086 ceph-mon[22689]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:31:21.764 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:31:21 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:31:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:31:21.765 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:21 smithi067 ceph-mon[21875]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:31:21.765 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:21 smithi067 ceph-mon[26633]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:31:23.764 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:23 smithi086 ceph-mon[22689]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:23.765 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:23 smithi067 ceph-mon[21875]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:23.765 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:23 smithi067 ceph-mon[26633]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:25.764 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:25 smithi086 ceph-mon[22689]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:25.765 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:25 smithi067 ceph-mon[21875]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:25.765 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:25 smithi067 ceph-mon[26633]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:27.764 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:27 smithi086 ceph-mon[22689]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:31:27.765 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:27 smithi067 ceph-mon[21875]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:31:27.765 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:27 smithi067 ceph-mon[26633]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:31:29.764 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:29 smithi086 ceph-mon[22689]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:29.765 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:29 smithi067 ceph-mon[21875]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:29.765 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:29 smithi067 ceph-mon[26633]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:30.764 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:30 smithi086 ceph-mon[22689]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:31:30.765 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:30 smithi067 ceph-mon[21875]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:31:30.766 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:30 smithi067 ceph-mon[26633]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:31:30.766 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:31:30 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:31:30] "GET /metrics HTTP/1.1" 200 207950 "" "Prometheus/2.33.4" 2024-04-22T22:31:31.764 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:31 smithi086 ceph-mon[22689]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:31:31.765 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:31:31 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:31:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:31:31.765 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:31 smithi067 ceph-mon[26633]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:31:31.766 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:31 smithi067 ceph-mon[21875]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:31:33.764 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:33 smithi086 ceph-mon[22689]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:33.765 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:33 smithi067 ceph-mon[21875]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:33.765 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:33 smithi067 ceph-mon[26633]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:35.764 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:35 smithi086 ceph-mon[22689]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:35.765 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:35 smithi067 ceph-mon[21875]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:35.766 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:35 smithi067 ceph-mon[26633]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:37.764 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:37 smithi086 ceph-mon[22689]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:31:37.765 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:37 smithi067 ceph-mon[21875]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:31:37.766 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:37 smithi067 ceph-mon[26633]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:31:39.764 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:39 smithi086 ceph-mon[22689]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:39.765 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:39 smithi067 ceph-mon[21875]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:39.766 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:39 smithi067 ceph-mon[26633]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:40.764 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:40 smithi086 ceph-mon[22689]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:31:40.765 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:40 smithi067 ceph-mon[21875]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:31:40.766 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:40 smithi067 ceph-mon[26633]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:31:40.766 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:31:40 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:31:40] "GET /metrics HTTP/1.1" 200 207950 "" "Prometheus/2.33.4" 2024-04-22T22:31:41.666 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:41 smithi086 ceph-mon[22689]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:31:41.666 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:31:41 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:31:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:31:41.766 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:41 smithi067 ceph-mon[21875]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:31:41.766 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:41 smithi067 ceph-mon[26633]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:31:43.765 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:43 smithi086 ceph-mon[22689]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:43.766 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:43 smithi067 ceph-mon[21875]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:43.767 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:43 smithi067 ceph-mon[26633]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:43.989 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -e sha1=430e09df97c8fc7dc2b2ae424f68ed11366c540f -- bash -c 'ceph orch ps' 2024-04-22T22:31:45.601 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:45 smithi067 ceph-mon[21875]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:45.602 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:45 smithi067 ceph-mon[26633]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:45.764 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:45 smithi086 ceph-mon[22689]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:46.123 INFO:teuthology.orchestra.run.smithi067.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-22T22:31:46.124 INFO:teuthology.orchestra.run.smithi067.stdout:alertmanager.a smithi067 *:9093,9094 running (6m) 3m ago 7m 25.9M - ba2b418f427c 9d0e1f18c1cb 2024-04-22T22:31:46.124 INFO:teuthology.orchestra.run.smithi067.stdout:grafana.a smithi086 *:3000 running (7m) 3m ago 7m 56.2M - 8.3.5 dad864ee21e9 5d6c83f8480c 2024-04-22T22:31:46.124 INFO:teuthology.orchestra.run.smithi067.stdout:iscsi.foo.smithi067.rrokcu smithi067 running (6m) 3m ago 6m 50.7M - 3.5 e1d6a67b021e 88568d9a3e4b 2024-04-22T22:31:46.124 INFO:teuthology.orchestra.run.smithi067.stdout:mgr.x smithi086 *:8443 running (11m) 3m ago 11m 412M - 17.2.0 e1d6a67b021e cd2bbd4855c3 2024-04-22T22:31:46.124 INFO:teuthology.orchestra.run.smithi067.stdout:mgr.y smithi067 *:9283 running (13m) 3m ago 13m 460M - 17.2.0 e1d6a67b021e c5a816fd6dde 2024-04-22T22:31:46.124 INFO:teuthology.orchestra.run.smithi067.stdout:mon.a smithi067 running (13m) 3m ago 13m 50.5M 2048M 17.2.0 e1d6a67b021e fbf59dc0efb6 2024-04-22T22:31:46.124 INFO:teuthology.orchestra.run.smithi067.stdout:mon.b smithi086 running (12m) 3m ago 12m 37.7M 2048M 17.2.0 e1d6a67b021e be2e0891a577 2024-04-22T22:31:46.124 INFO:teuthology.orchestra.run.smithi067.stdout:mon.c smithi067 running (12m) 3m ago 12m 40.3M 2048M 17.2.0 e1d6a67b021e e93c724294c9 2024-04-22T22:31:46.124 INFO:teuthology.orchestra.run.smithi067.stdout:node-exporter.a smithi067 *:9100 running (8m) 3m ago 8m 15.1M - 1dbe0e931976 865795666bfc 2024-04-22T22:31:46.124 INFO:teuthology.orchestra.run.smithi067.stdout:node-exporter.b smithi086 *:9100 running (8m) 3m ago 8m 14.7M - 1dbe0e931976 bc12658d1c9c 2024-04-22T22:31:46.124 INFO:teuthology.orchestra.run.smithi067.stdout:osd.0 smithi067 running (11m) 3m ago 11m 50.9M 4016M 17.2.0 e1d6a67b021e 2774168bdf21 2024-04-22T22:31:46.124 INFO:teuthology.orchestra.run.smithi067.stdout:osd.1 smithi067 running (11m) 3m ago 11m 51.9M 4016M 17.2.0 e1d6a67b021e c07f4f56fb85 2024-04-22T22:31:46.124 INFO:teuthology.orchestra.run.smithi067.stdout:osd.2 smithi067 running (10m) 3m ago 10m 48.0M 4016M 17.2.0 e1d6a67b021e 5431738fc1b7 2024-04-22T22:31:46.125 INFO:teuthology.orchestra.run.smithi067.stdout:osd.3 smithi067 running (10m) 3m ago 10m 48.2M 4016M 17.2.0 e1d6a67b021e ea1ecfb9c931 2024-04-22T22:31:46.125 INFO:teuthology.orchestra.run.smithi067.stdout:osd.4 smithi086 running (10m) 3m ago 10m 48.3M 4272M 17.2.0 e1d6a67b021e 4196c45a856b 2024-04-22T22:31:46.125 INFO:teuthology.orchestra.run.smithi067.stdout:osd.5 smithi086 running (9m) 3m ago 9m 47.4M 4272M 17.2.0 e1d6a67b021e 6c685d1d93a3 2024-04-22T22:31:46.125 INFO:teuthology.orchestra.run.smithi067.stdout:osd.6 smithi086 running (9m) 3m ago 9m 46.8M 4272M 17.2.0 e1d6a67b021e 7e586ba77fac 2024-04-22T22:31:46.125 INFO:teuthology.orchestra.run.smithi067.stdout:osd.7 smithi086 running (8m) 3m ago 8m 48.0M 4272M 17.2.0 e1d6a67b021e fa3c88c96bae 2024-04-22T22:31:46.125 INFO:teuthology.orchestra.run.smithi067.stdout:prometheus.a smithi086 *:9095 running (6m) 3m ago 8m 64.1M - 514e6a882f6e 8531ba0d81dc 2024-04-22T22:31:46.125 INFO:teuthology.orchestra.run.smithi067.stdout:rgw.foo.smithi067.bekyzf smithi067 *:8000 running (6m) 3m ago 6m 71.4M - 17.2.0 e1d6a67b021e df1f503ccd1b 2024-04-22T22:31:46.125 INFO:teuthology.orchestra.run.smithi067.stdout:rgw.foo.smithi086.cofxku smithi086 *:8000 running (6m) 3m ago 6m 71.1M - 17.2.0 e1d6a67b021e 66e404643e32 2024-04-22T22:31:46.756 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -e sha1=430e09df97c8fc7dc2b2ae424f68ed11366c540f -- bash -c 'ceph versions' 2024-04-22T22:31:47.755 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:47 smithi067 ceph-mon[21875]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:31:47.756 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:47 smithi067 ceph-mon[21875]: from='client.15024 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:31:47.756 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:47 smithi067 ceph-mon[26633]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:31:47.756 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:47 smithi067 ceph-mon[26633]: from='client.15024 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:31:47.764 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:47 smithi086 ceph-mon[22689]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:31:47.764 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:47 smithi086 ceph-mon[22689]: from='client.15024 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-22T22:31:48.743 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:48 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:31:48.743 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:48 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:31:48.743 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:48 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:31:48.743 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:48 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T22:31:48.743 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:48 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T22:31:48.743 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:48 smithi067 ceph-mon[21875]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:31:48.743 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:48 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:31:48.744 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:48 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:31:48.744 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:48 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:31:48.744 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:48 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T22:31:48.744 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:48 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T22:31:48.744 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:48 smithi067 ceph-mon[26633]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:31:48.764 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:48 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:31:48.765 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:48 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-22T22:31:48.765 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:48 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' 2024-04-22T22:31:48.765 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:48 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-22T22:31:48.765 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:48 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-22T22:31:48.765 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:48 smithi086 ceph-mon[22689]: from='mgr.14451 172.21.15.67:0/3610014973' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-22T22:31:48.969 INFO:teuthology.orchestra.run.smithi067.stdout:{ 2024-04-22T22:31:48.969 INFO:teuthology.orchestra.run.smithi067.stdout: "mon": { 2024-04-22T22:31:48.969 INFO:teuthology.orchestra.run.smithi067.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-22T22:31:48.969 INFO:teuthology.orchestra.run.smithi067.stdout: }, 2024-04-22T22:31:48.969 INFO:teuthology.orchestra.run.smithi067.stdout: "mgr": { 2024-04-22T22:31:48.969 INFO:teuthology.orchestra.run.smithi067.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T22:31:48.970 INFO:teuthology.orchestra.run.smithi067.stdout: }, 2024-04-22T22:31:48.970 INFO:teuthology.orchestra.run.smithi067.stdout: "osd": { 2024-04-22T22:31:48.970 INFO:teuthology.orchestra.run.smithi067.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-22T22:31:48.970 INFO:teuthology.orchestra.run.smithi067.stdout: }, 2024-04-22T22:31:48.970 INFO:teuthology.orchestra.run.smithi067.stdout: "mds": {}, 2024-04-22T22:31:48.970 INFO:teuthology.orchestra.run.smithi067.stdout: "rgw": { 2024-04-22T22:31:48.970 INFO:teuthology.orchestra.run.smithi067.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-22T22:31:48.970 INFO:teuthology.orchestra.run.smithi067.stdout: }, 2024-04-22T22:31:48.970 INFO:teuthology.orchestra.run.smithi067.stdout: "overall": { 2024-04-22T22:31:48.970 INFO:teuthology.orchestra.run.smithi067.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 15 2024-04-22T22:31:48.970 INFO:teuthology.orchestra.run.smithi067.stdout: } 2024-04-22T22:31:48.970 INFO:teuthology.orchestra.run.smithi067.stdout:} 2024-04-22T22:31:49.580 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:49 smithi067 ceph-mon[21875]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:49.580 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:49 smithi067 ceph-mon[21875]: Deploying daemon mgr.x on smithi086 2024-04-22T22:31:49.581 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:49 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/722394843' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T22:31:49.581 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:49 smithi067 ceph-mon[26633]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:49.581 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:49 smithi067 ceph-mon[26633]: Deploying daemon mgr.x on smithi086 2024-04-22T22:31:49.581 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:49 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/722394843' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T22:31:49.607 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -e sha1=430e09df97c8fc7dc2b2ae424f68ed11366c540f -- bash -c 'ceph -s' 2024-04-22T22:31:49.765 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:49 smithi086 ceph-mon[22689]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:49.765 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:49 smithi086 ceph-mon[22689]: Deploying daemon mgr.x on smithi086 2024-04-22T22:31:49.765 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:49 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/722394843' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T22:31:50.516 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:50 smithi067 ceph-mon[21875]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:31:50.516 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:50 smithi067 ceph-mon[26633]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:31:50.764 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:50 smithi086 ceph-mon[22689]: from='client.14829 -' entity='client.iscsi.foo.smithi067.rrokcu' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-22T22:31:50.954 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:31:50 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:31:50] "GET /metrics HTTP/1.1" 200 207948 "" "Prometheus/2.33.4" 2024-04-22T22:31:51.516 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:51 smithi067 ceph-mon[21875]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:31:51.516 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:51 smithi067 ceph-mon[26633]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:31:51.688 INFO:teuthology.orchestra.run.smithi067.stdout: cluster: 2024-04-22T22:31:51.688 INFO:teuthology.orchestra.run.smithi067.stdout: id: 1ab1a3a8-00f6-11ef-bc93-c7b262605968 2024-04-22T22:31:51.688 INFO:teuthology.orchestra.run.smithi067.stdout: health: HEALTH_OK 2024-04-22T22:31:51.688 INFO:teuthology.orchestra.run.smithi067.stdout: 2024-04-22T22:31:51.688 INFO:teuthology.orchestra.run.smithi067.stdout: services: 2024-04-22T22:31:51.689 INFO:teuthology.orchestra.run.smithi067.stdout: mon: 3 daemons, quorum a,c,b (age 12m) 2024-04-22T22:31:51.689 INFO:teuthology.orchestra.run.smithi067.stdout: mgr: y(active, since 8m), standbys: x 2024-04-22T22:31:51.689 INFO:teuthology.orchestra.run.smithi067.stdout: osd: 8 osds: 8 up (since 8m), 8 in (since 9m) 2024-04-22T22:31:51.689 INFO:teuthology.orchestra.run.smithi067.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-04-22T22:31:51.689 INFO:teuthology.orchestra.run.smithi067.stdout: 2024-04-22T22:31:51.689 INFO:teuthology.orchestra.run.smithi067.stdout: data: 2024-04-22T22:31:51.689 INFO:teuthology.orchestra.run.smithi067.stdout: pools: 6 pools, 161 pgs 2024-04-22T22:31:51.689 INFO:teuthology.orchestra.run.smithi067.stdout: objects: 209 objects, 585 KiB 2024-04-22T22:31:51.689 INFO:teuthology.orchestra.run.smithi067.stdout: usage: 64 MiB used, 715 GiB / 715 GiB avail 2024-04-22T22:31:51.689 INFO:teuthology.orchestra.run.smithi067.stdout: pgs: 161 active+clean 2024-04-22T22:31:51.689 INFO:teuthology.orchestra.run.smithi067.stdout: 2024-04-22T22:31:51.689 INFO:teuthology.orchestra.run.smithi067.stdout: io: 2024-04-22T22:31:51.689 INFO:teuthology.orchestra.run.smithi067.stdout: client: 1.2 KiB/s rd, 1 op/s rd, 0 op/s wr 2024-04-22T22:31:51.689 INFO:teuthology.orchestra.run.smithi067.stdout: 2024-04-22T22:31:51.765 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:51 smithi086 ceph-mon[22689]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:31:51.765 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:31:51 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:31:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:31:52.337 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -e sha1=430e09df97c8fc7dc2b2ae424f68ed11366c540f -- bash -c 'ceph health detail' 2024-04-22T22:31:52.652 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:52 smithi067 ceph-mon[21875]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:52.652 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:52 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/1963040751' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-22T22:31:52.653 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:52 smithi067 ceph-mon[26633]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:52.653 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:52 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/1963040751' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-22T22:31:52.764 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:52 smithi086 ceph-mon[22689]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:52.765 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:52 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/1963040751' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-22T22:31:54.512 INFO:teuthology.orchestra.run.smithi067.stdout:HEALTH_OK 2024-04-22T22:31:54.766 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:54 smithi067 ceph-mon[21875]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:54.766 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:54 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/1989261746' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T22:31:54.766 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:54 smithi067 ceph-mon[26633]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:54.766 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:54 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/1989261746' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T22:31:55.015 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:54 smithi086 ceph-mon[22689]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-22T22:31:55.015 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:54 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/1989261746' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-22T22:31:55.166 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -e sha1=430e09df97c8fc7dc2b2ae424f68ed11366c540f -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 2'"'"'' 2024-04-22T22:31:57.260 INFO:teuthology.orchestra.run.smithi067.stdout:false 2024-04-22T22:31:57.261 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:56 smithi067 ceph-mon[21875]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:31:57.261 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:56 smithi067 ceph-mon[26633]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:31:57.265 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:56 smithi086 ceph-mon[22689]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-22T22:31:57.839 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:57 smithi067 ceph-mon[21875]: from='client.? 172.21.15.67:0/4070157190' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T22:31:57.839 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:57 smithi067 ceph-mon[26633]: from='client.? 172.21.15.67:0/4070157190' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T22:31:57.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T22:31:57.863 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_c66b8bf2efd3f3988ac1851474c2f98eb2ca30d9/qa/tasks/cephadm.py", line 1469, in shell _shell( File "/home/teuthworker/src/git.ceph.com_ceph_c66b8bf2efd3f3988ac1851474c2f98eb2ca30d9/qa/tasks/cephadm.py", line 110, in _shell return remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi067 with status 1: 'sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -e sha1=430e09df97c8fc7dc2b2ae424f68ed11366c540f -- bash -c \'ceph versions | jq -e \'"\'"\'.mgr | length == 2\'"\'"\'\'' 2024-04-22T22:31:58.079 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=082249e5b7704c90b5a2c8ede1d47f94 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_c66b8bf2efd3f3988ac1851474c2f98eb2ca30d9/qa/tasks/cephadm.py", line 1469, in shell _shell( File "/home/teuthworker/src/git.ceph.com_ceph_c66b8bf2efd3f3988ac1851474c2f98eb2ca30d9/qa/tasks/cephadm.py", line 110, in _shell return remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi067 with status 1: 'sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -e sha1=430e09df97c8fc7dc2b2ae424f68ed11366c540f -- bash -c \'ceph versions | jq -e \'"\'"\'.mgr | length == 2\'"\'"\'\'' 2024-04-22T22:31:58.082 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-04-22T22:31:58.091 INFO:tasks.cephadm:Teardown begin 2024-04-22T22:31:58.092 DEBUG:teuthology.orchestra.run.smithi067:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-22T22:31:58.123 DEBUG:teuthology.orchestra.run.smithi086:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-22T22:31:58.149 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:31:57 smithi086 ceph-mon[22689]: from='client.? 172.21.15.67:0/4070157190' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-22T22:31:58.154 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-04-22T22:31:58.154 DEBUG:teuthology.orchestra.run.smithi067:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-22T22:31:58.185 DEBUG:teuthology.orchestra.run.smithi086:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-22T22:31:58.213 INFO:tasks.cephadm:Stopping all daemons... 2024-04-22T22:31:58.213 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-04-22T22:31:58.214 DEBUG:teuthology.orchestra.run.smithi067:> sudo systemctl stop ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@mon.a 2024-04-22T22:31:58.452 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:58 smithi067 systemd[1]: Stopping Ceph mon.a for 1ab1a3a8-00f6-11ef-bc93-c7b262605968... 2024-04-22T22:31:58.452 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:58 smithi067 bash[53642]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mon.a" found: no such container 2024-04-22T22:31:58.766 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:58 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mon-a[21871]: 2024-04-22T22:31:58.453+0000 7fd68de1e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T22:31:58.766 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:58 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mon-a[21871]: 2024-04-22T22:31:58.453+0000 7fd68de1e700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-04-22T22:31:58.766 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:58 smithi067 podman[53653]: 2024-04-22 22:31:58.47408322 +0000 UTC m=+0.164412271 container died fbf59dc0efb64c6a0df5907527f99215cee7b1d16f837b5b62e7cf0450d0b1df (image=quay.io/ceph/ceph:v17.2.0, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mon-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, ceph=True, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, version=8, distribution-scope=public, vcs-type=git, release=754, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, architecture=x86_64) 2024-04-22T22:31:59.110 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:58 smithi067 podman[53653]: 2024-04-22 22:31:58.869962403 +0000 UTC m=+0.560291470 container cleanup fbf59dc0efb64c6a0df5907527f99215cee7b1d16f837b5b62e7cf0450d0b1df (image=quay.io/ceph/ceph:v17.2.0, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mon-a, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, ceph=True, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., vcs-type=git, architecture=x86_64, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, version=8, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-22T22:31:59.111 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:58 smithi067 bash[53653]: ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mon-a 2024-04-22T22:31:59.111 INFO:journalctl@ceph.mon.a.smithi067.stdout:Apr 22 22:31:58 smithi067 bash[53680]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mon.a" found: no such container 2024-04-22T22:31:59.116 DEBUG:teuthology.orchestra.run.smithi067:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@mon.a.service' 2024-04-22T22:31:59.158 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T22:31:59.158 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-04-22T22:31:59.159 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-04-22T22:31:59.159 DEBUG:teuthology.orchestra.run.smithi067:> sudo systemctl stop ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@mon.c 2024-04-22T22:31:59.482 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:59 smithi067 systemd[1]: Stopping Ceph mon.c for 1ab1a3a8-00f6-11ef-bc93-c7b262605968... 2024-04-22T22:31:59.482 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:59 smithi067 bash[53735]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mon.c" found: no such container 2024-04-22T22:31:59.766 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:59 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mon-c[26629]: 2024-04-22T22:31:59.482+0000 7f1e35740700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T22:31:59.767 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:59 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mon-c[26629]: 2024-04-22T22:31:59.482+0000 7f1e35740700 -1 mon.c@1(peon) e3 *** Got Signal Terminated *** 2024-04-22T22:31:59.767 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:59 smithi067 podman[53748]: 2024-04-22 22:31:59.6369881 +0000 UTC m=+0.360903620 container died e93c724294c9a6ae8393005e1168e35745b8f405fe7fb7fd3285591a5906b519 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mon-c, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, name=centos-stream, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, distribution-scope=public, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, com.redhat.component=centos-stream-container, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-22T22:32:00.182 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:59 smithi067 podman[53748]: 2024-04-22 22:31:59.983065102 +0000 UTC m=+0.706980619 container cleanup e93c724294c9a6ae8393005e1168e35745b8f405fe7fb7fd3285591a5906b519 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mon-c, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, name=centos-stream, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, io.openshift.expose-services=, release=754, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-22T22:32:00.182 INFO:journalctl@ceph.mon.c.smithi067.stdout:Apr 22 22:31:59 smithi067 bash[53748]: ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mon-c 2024-04-22T22:32:00.201 DEBUG:teuthology.orchestra.run.smithi067:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@mon.c.service' 2024-04-22T22:32:00.239 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T22:32:00.240 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-04-22T22:32:00.240 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-04-22T22:32:00.240 DEBUG:teuthology.orchestra.run.smithi086:> sudo systemctl stop ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@mon.b 2024-04-22T22:32:00.765 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:32:00 smithi086 systemd[1]: Stopping Ceph mon.b for 1ab1a3a8-00f6-11ef-bc93-c7b262605968... 2024-04-22T22:32:00.765 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:32:00 smithi086 bash[43171]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mon.b" found: no such container 2024-04-22T22:32:00.765 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:32:00 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mon-b[22685]: 2024-04-22T22:32:00.417+0000 7ff15e1d0700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T22:32:00.765 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:32:00 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mon-b[22685]: 2024-04-22T22:32:00.417+0000 7ff15e1d0700 -1 mon.b@2(peon) e3 *** Got Signal Terminated *** 2024-04-22T22:32:00.765 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:32:00 smithi086 podman[43184]: 2024-04-22 22:32:00.46880486 +0000 UTC m=+0.142253944 container died be2e0891a57742b32181076130b29486a006138ea30755751c260c73b148f9f4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mon-b, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, name=centos-stream, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, release=754, version=8, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, vcs-type=git, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, ceph=True) 2024-04-22T22:32:01.016 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:32:00 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:32:00] "GET /metrics HTTP/1.1" 200 207954 "" "Prometheus/2.33.4" 2024-04-22T22:32:01.168 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:32:00 smithi086 podman[43184]: 2024-04-22 22:32:00.935779177 +0000 UTC m=+0.609228260 container cleanup be2e0891a57742b32181076130b29486a006138ea30755751c260c73b148f9f4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mon-b, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, io.buildah.version=1.19.8, io.openshift.expose-services=, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, maintainer=Guillaume Abrioux , RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, com.redhat.component=centos-stream-container) 2024-04-22T22:32:01.169 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:32:00 smithi086 bash[43184]: ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mon-b 2024-04-22T22:32:01.169 INFO:journalctl@ceph.mon.b.smithi086.stdout:Apr 22 22:32:00 smithi086 bash[43211]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mon.b" found: no such container 2024-04-22T22:32:01.174 DEBUG:teuthology.orchestra.run.smithi086:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@mon.b.service' 2024-04-22T22:32:01.221 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T22:32:01.222 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-04-22T22:32:01.222 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-04-22T22:32:01.222 DEBUG:teuthology.orchestra.run.smithi067:> sudo systemctl stop ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@mgr.y 2024-04-22T22:32:01.516 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:32:01 smithi067 systemd[1]: Stopping Ceph mgr.y for 1ab1a3a8-00f6-11ef-bc93-c7b262605968... 2024-04-22T22:32:01.516 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:32:01 smithi067 bash[53825]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr.y" found: no such container 2024-04-22T22:32:01.517 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:32:01 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:32:01.445+0000 7fe1475a1700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T22:32:01.517 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:32:01 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y[22102]: 2024-04-22T22:32:01.445+0000 7fe1475a1700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-04-22T22:32:01.517 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:32:01 smithi067 podman[53835]: 2024-04-22 22:32:01.477190716 +0000 UTC m=+0.172467019 container died c5a816fd6ddeadf20fc00707d927f83d3e307f72801ab39536be5b0e0ec82140 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, io.openshift.expose-services=, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, io.openshift.tags=base centos centos-stream, ceph=True, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , vcs-type=git, io.k8s.display-name=CentOS Stream 8, name=centos-stream, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-22T22:32:01.765 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:32:01 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x[23760]: ::ffff:172.21.15.86 - - [22/Apr/2024:22:32:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-22T22:32:02.128 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:32:01 smithi067 podman[53835]: 2024-04-22 22:32:01.879080782 +0000 UTC m=+0.574357096 container cleanup c5a816fd6ddeadf20fc00707d927f83d3e307f72801ab39536be5b0e0ec82140 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, vcs-type=git, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True) 2024-04-22T22:32:02.128 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:32:01 smithi067 bash[53835]: ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-y 2024-04-22T22:32:02.128 INFO:journalctl@ceph.mgr.y.smithi067.stdout:Apr 22 22:32:01 smithi067 bash[53861]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr.y" found: no such container 2024-04-22T22:32:02.135 DEBUG:teuthology.orchestra.run.smithi067:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@mgr.y.service' 2024-04-22T22:32:02.178 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T22:32:02.178 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-04-22T22:32:02.178 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-04-22T22:32:02.178 DEBUG:teuthology.orchestra.run.smithi086:> sudo systemctl stop ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@mgr.x 2024-04-22T22:32:02.515 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:32:02 smithi086 systemd[1]: Stopping Ceph mgr.x for 1ab1a3a8-00f6-11ef-bc93-c7b262605968... 2024-04-22T22:32:02.515 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:32:02 smithi086 bash[43266]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr.x" found: no such container 2024-04-22T22:32:02.515 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:32:02 smithi086 podman[43279]: 2024-04-22 22:32:02.394848446 +0000 UTC m=+0.137843536 container died cd2bbd4855c30e295ff6e20cbc28be97add0724dbb4503ac4c9859f0f44861b8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x, release=754, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, version=8, maintainer=Guillaume Abrioux , name=centos-stream, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-22T22:32:02.924 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:32:02 smithi086 podman[43279]: 2024-04-22 22:32:02.74058216 +0000 UTC m=+0.483577250 container cleanup cd2bbd4855c30e295ff6e20cbc28be97add0724dbb4503ac4c9859f0f44861b8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x, distribution-scope=public, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, name=centos-stream, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-22T22:32:02.924 INFO:journalctl@ceph.mgr.x.smithi086.stdout:Apr 22 22:32:02 smithi086 bash[43279]: ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-mgr-x 2024-04-22T22:32:02.965 DEBUG:teuthology.orchestra.run.smithi086:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@mgr.x.service' 2024-04-22T22:32:03.000 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T22:32:03.000 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-04-22T22:32:03.000 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-04-22T22:32:03.000 DEBUG:teuthology.orchestra.run.smithi067:> sudo systemctl stop ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@osd.0 2024-04-22T22:32:03.516 INFO:journalctl@ceph.osd.0.smithi067.stdout:Apr 22 22:32:03 smithi067 systemd[1]: Stopping Ceph osd.0 for 1ab1a3a8-00f6-11ef-bc93-c7b262605968... 2024-04-22T22:32:03.516 INFO:journalctl@ceph.osd.0.smithi067.stdout:Apr 22 22:32:03 smithi067 bash[53916]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd.0" found: no such container 2024-04-22T22:32:03.516 INFO:journalctl@ceph.osd.0.smithi067.stdout:Apr 22 22:32:03 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-0[29486]: 2024-04-22T22:32:03.217+0000 7f41647d0700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T22:32:03.517 INFO:journalctl@ceph.osd.0.smithi067.stdout:Apr 22 22:32:03 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-0[29486]: 2024-04-22T22:32:03.217+0000 7f41647d0700 -1 osd.0 95 *** Got signal Terminated *** 2024-04-22T22:32:03.517 INFO:journalctl@ceph.osd.0.smithi067.stdout:Apr 22 22:32:03 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-0[29486]: 2024-04-22T22:32:03.217+0000 7f41647d0700 -1 osd.0 95 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T22:32:08.924 INFO:journalctl@ceph.osd.0.smithi067.stdout:Apr 22 22:32:08 smithi067 podman[53929]: 2024-04-22 22:32:08.642433384 +0000 UTC m=+5.556794286 container died 2774168bdf217f63467c01464ae02866243c8440c78cb1150c9e719cdc87405c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, vcs-type=git, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, distribution-scope=public, release=754, ceph=True, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream) 2024-04-22T22:32:09.266 INFO:journalctl@ceph.osd.0.smithi067.stdout:Apr 22 22:32:09 smithi067 podman[53929]: 2024-04-22 22:32:09.008636549 +0000 UTC m=+5.922997448 container cleanup 2774168bdf217f63467c01464ae02866243c8440c78cb1150c9e719cdc87405c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-0, GIT_BRANCH=HEAD, release=754, GIT_CLEAN=True, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, ceph=True, distribution-scope=public, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, maintainer=Guillaume Abrioux , name=centos-stream, io.k8s.display-name=CentOS Stream 8, vcs-type=git, RELEASE=HEAD, vendor=Red Hat, Inc.) 2024-04-22T22:32:09.266 INFO:journalctl@ceph.osd.0.smithi067.stdout:Apr 22 22:32:09 smithi067 bash[53929]: ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-0 2024-04-22T22:32:09.266 INFO:journalctl@ceph.osd.0.smithi067.stdout:Apr 22 22:32:09 smithi067 bash[53959]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd.0" found: no such container 2024-04-22T22:32:09.673 INFO:journalctl@ceph.osd.0.smithi067.stdout:Apr 22 22:32:09 smithi067 podman[53946]: 2024-04-22 22:32:09.326779992 +0000 UTC m=+0.675137434 container remove 2774168bdf217f63467c01464ae02866243c8440c78cb1150c9e719cdc87405c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-0, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, ceph=True, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, com.redhat.component=centos-stream-container, architecture=x86_64, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, maintainer=Guillaume Abrioux ) 2024-04-22T22:32:09.674 INFO:journalctl@ceph.osd.0.smithi067.stdout:Apr 22 22:32:09 smithi067 podman[54024]: 2024-04-22 22:32:09.40531051 +0000 UTC m=+0.019520823 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-22T22:32:09.674 INFO:journalctl@ceph.osd.0.smithi067.stdout:Apr 22 22:32:09 smithi067 podman[54024]: 2024-04-22 22:32:09.52759554 +0000 UTC m=+0.141805860 container create e8d29013e22a4cc145b49225124fa8fd4f7f747d8107e09179bb526dc23e3d11 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-0-deactivate, name=centos-stream, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, release=754, GIT_BRANCH=HEAD, version=8) 2024-04-22T22:32:10.016 INFO:journalctl@ceph.osd.0.smithi067.stdout:Apr 22 22:32:09 smithi067 podman[54024]: 2024-04-22 22:32:09.788339458 +0000 UTC m=+0.402549748 container init e8d29013e22a4cc145b49225124fa8fd4f7f747d8107e09179bb526dc23e3d11 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-0-deactivate, com.redhat.component=centos-stream-container, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, RELEASE=HEAD, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, maintainer=Guillaume Abrioux , distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8) 2024-04-22T22:32:10.016 INFO:journalctl@ceph.osd.0.smithi067.stdout:Apr 22 22:32:09 smithi067 podman[54024]: 2024-04-22 22:32:09.791779029 +0000 UTC m=+0.405989312 container start e8d29013e22a4cc145b49225124fa8fd4f7f747d8107e09179bb526dc23e3d11 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-0-deactivate, version=8, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, ceph=True, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-22T22:32:10.017 INFO:journalctl@ceph.osd.0.smithi067.stdout:Apr 22 22:32:09 smithi067 podman[54024]: 2024-04-22 22:32:09.888291191 +0000 UTC m=+0.502501484 container attach e8d29013e22a4cc145b49225124fa8fd4f7f747d8107e09179bb526dc23e3d11 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-0-deactivate, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, distribution-scope=public, release=754, io.k8s.display-name=CentOS Stream 8, version=8, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream) 2024-04-22T22:32:10.439 INFO:journalctl@ceph.osd.0.smithi067.stdout:Apr 22 22:32:10 smithi067 podman[54024]: 2024-04-22 22:32:10.090508529 +0000 UTC m=+0.704718824 container died e8d29013e22a4cc145b49225124fa8fd4f7f747d8107e09179bb526dc23e3d11 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, com.redhat.component=centos-stream-container, release=754) 2024-04-22T22:32:10.731 DEBUG:teuthology.orchestra.run.smithi067:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@osd.0.service' 2024-04-22T22:32:10.773 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T22:32:10.773 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-04-22T22:32:10.773 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-04-22T22:32:10.774 DEBUG:teuthology.orchestra.run.smithi067:> sudo systemctl stop ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@osd.1 2024-04-22T22:32:10.985 INFO:journalctl@ceph.osd.1.smithi067.stdout:Apr 22 22:32:10 smithi067 systemd[1]: Stopping Ceph osd.1 for 1ab1a3a8-00f6-11ef-bc93-c7b262605968... 2024-04-22T22:32:10.985 INFO:journalctl@ceph.osd.1.smithi067.stdout:Apr 22 22:32:10 smithi067 bash[54101]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd.1" found: no such container 2024-04-22T22:32:11.266 INFO:journalctl@ceph.osd.1.smithi067.stdout:Apr 22 22:32:10 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-1[32457]: 2024-04-22T22:32:10.985+0000 7fc0df3c4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T22:32:11.266 INFO:journalctl@ceph.osd.1.smithi067.stdout:Apr 22 22:32:10 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-1[32457]: 2024-04-22T22:32:10.985+0000 7fc0df3c4700 -1 osd.1 95 *** Got signal Terminated *** 2024-04-22T22:32:11.267 INFO:journalctl@ceph.osd.1.smithi067.stdout:Apr 22 22:32:10 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-1[32457]: 2024-04-22T22:32:10.985+0000 7fc0df3c4700 -1 osd.1 95 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T22:32:16.766 INFO:journalctl@ceph.osd.1.smithi067.stdout:Apr 22 22:32:16 smithi067 podman[54112]: 2024-04-22 22:32:16.491506137 +0000 UTC m=+5.605069451 container died c07f4f56fb85e846ab940b36641079a98c351318bfe951e489bbcceecbd314fb (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-1, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, vcs-type=git, version=8, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, name=centos-stream, architecture=x86_64, io.buildah.version=1.19.8, release=754, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-22T22:32:17.119 INFO:journalctl@ceph.osd.1.smithi067.stdout:Apr 22 22:32:16 smithi067 podman[54112]: 2024-04-22 22:32:16.852962731 +0000 UTC m=+5.966526044 container cleanup c07f4f56fb85e846ab940b36641079a98c351318bfe951e489bbcceecbd314fb (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-1, GIT_BRANCH=HEAD, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, ceph=True, RELEASE=HEAD, name=centos-stream, release=754, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8) 2024-04-22T22:32:17.119 INFO:journalctl@ceph.osd.1.smithi067.stdout:Apr 22 22:32:16 smithi067 bash[54112]: ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-1 2024-04-22T22:32:17.119 INFO:journalctl@ceph.osd.1.smithi067.stdout:Apr 22 22:32:16 smithi067 bash[54143]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd.1" found: no such container 2024-04-22T22:32:17.462 INFO:journalctl@ceph.osd.1.smithi067.stdout:Apr 22 22:32:17 smithi067 podman[54131]: 2024-04-22 22:32:17.120706834 +0000 UTC m=+0.619545623 container remove c07f4f56fb85e846ab940b36641079a98c351318bfe951e489bbcceecbd314fb (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-1, io.openshift.expose-services=, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , RELEASE=HEAD, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, io.openshift.tags=base centos centos-stream, architecture=x86_64, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-22T22:32:17.462 INFO:journalctl@ceph.osd.1.smithi067.stdout:Apr 22 22:32:17 smithi067 podman[54205]: 2024-04-22 22:32:17.209797494 +0000 UTC m=+0.019439278 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-22T22:32:17.462 INFO:journalctl@ceph.osd.1.smithi067.stdout:Apr 22 22:32:17 smithi067 podman[54205]: 2024-04-22 22:32:17.329876071 +0000 UTC m=+0.139517827 container create e49766532a203104008bbff017a8f903f339bed84743d5600e63feea7dc35e29 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-1-deactivate, io.openshift.tags=base centos centos-stream, architecture=x86_64, RELEASE=HEAD, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, distribution-scope=public, name=centos-stream, release=754, build-date=2022-05-03T08:36:31.336870, version=8, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-22T22:32:17.766 INFO:journalctl@ceph.osd.1.smithi067.stdout:Apr 22 22:32:17 smithi067 podman[54205]: 2024-04-22 22:32:17.54896789 +0000 UTC m=+0.358609659 container init e49766532a203104008bbff017a8f903f339bed84743d5600e63feea7dc35e29 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=-17.2.0, ceph=True, RELEASE=HEAD, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, release=754, distribution-scope=public, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, name=centos-stream, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., version=8, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-04-22T22:32:17.767 INFO:journalctl@ceph.osd.1.smithi067.stdout:Apr 22 22:32:17 smithi067 podman[54205]: 2024-04-22 22:32:17.552111925 +0000 UTC m=+0.361753681 container start e49766532a203104008bbff017a8f903f339bed84743d5600e63feea7dc35e29 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-1-deactivate, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, vcs-type=git, GIT_CLEAN=True, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, name=centos-stream, ceph=True, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-22T22:32:17.767 INFO:journalctl@ceph.osd.1.smithi067.stdout:Apr 22 22:32:17 smithi067 podman[54205]: 2024-04-22 22:32:17.632108223 +0000 UTC m=+0.441749994 container attach e49766532a203104008bbff017a8f903f339bed84743d5600e63feea7dc35e29 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-1-deactivate, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, GIT_CLEAN=True, distribution-scope=public, ceph=True, com.redhat.component=centos-stream-container, vcs-type=git, name=centos-stream, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, io.openshift.expose-services=, architecture=x86_64, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream) 2024-04-22T22:32:18.133 INFO:journalctl@ceph.osd.1.smithi067.stdout:Apr 22 22:32:17 smithi067 podman[54205]: 2024-04-22 22:32:17.844254964 +0000 UTC m=+0.653896723 container died e49766532a203104008bbff017a8f903f339bed84743d5600e63feea7dc35e29 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-1-deactivate, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, name=centos-stream, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, version=8, io.k8s.display-name=CentOS Stream 8, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-22T22:32:18.422 DEBUG:teuthology.orchestra.run.smithi067:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@osd.1.service' 2024-04-22T22:32:18.464 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T22:32:18.464 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-04-22T22:32:18.464 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-04-22T22:32:18.465 DEBUG:teuthology.orchestra.run.smithi067:> sudo systemctl stop ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@osd.2 2024-04-22T22:32:18.664 INFO:journalctl@ceph.osd.2.smithi067.stdout:Apr 22 22:32:18 smithi067 systemd[1]: Stopping Ceph osd.2 for 1ab1a3a8-00f6-11ef-bc93-c7b262605968... 2024-04-22T22:32:18.665 INFO:journalctl@ceph.osd.2.smithi067.stdout:Apr 22 22:32:18 smithi067 bash[54285]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd.2" found: no such container 2024-04-22T22:32:19.016 INFO:journalctl@ceph.osd.2.smithi067.stdout:Apr 22 22:32:18 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-2[35377]: 2024-04-22T22:32:18.698+0000 7fee8bec1700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T22:32:19.016 INFO:journalctl@ceph.osd.2.smithi067.stdout:Apr 22 22:32:18 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-2[35377]: 2024-04-22T22:32:18.698+0000 7fee8bec1700 -1 osd.2 95 *** Got signal Terminated *** 2024-04-22T22:32:19.017 INFO:journalctl@ceph.osd.2.smithi067.stdout:Apr 22 22:32:18 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-2[35377]: 2024-04-22T22:32:18.698+0000 7fee8bec1700 -1 osd.2 95 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T22:32:24.312 INFO:journalctl@ceph.osd.2.smithi067.stdout:Apr 22 22:32:24 smithi067 podman[54296]: 2024-04-22 22:32:24.060976874 +0000 UTC m=+5.481818120 container died 5431738fc1b74708207e32fdd1bbbed0a2462c4840ddce2ea29bf2589d5d9501 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-2, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , release=754, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, distribution-scope=public, com.redhat.component=centos-stream-container, ceph=True, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, GIT_CLEAN=True) 2024-04-22T22:32:24.712 INFO:journalctl@ceph.osd.2.smithi067.stdout:Apr 22 22:32:24 smithi067 podman[54296]: 2024-04-22 22:32:24.413551171 +0000 UTC m=+5.834392402 container cleanup 5431738fc1b74708207e32fdd1bbbed0a2462c4840ddce2ea29bf2589d5d9501 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-2, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, name=centos-stream, vcs-type=git, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , version=8, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, distribution-scope=public, release=754, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870) 2024-04-22T22:32:24.713 INFO:journalctl@ceph.osd.2.smithi067.stdout:Apr 22 22:32:24 smithi067 bash[54296]: ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-2 2024-04-22T22:32:24.713 INFO:journalctl@ceph.osd.2.smithi067.stdout:Apr 22 22:32:24 smithi067 bash[54337]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd.2" found: no such container 2024-04-22T22:32:25.016 INFO:journalctl@ceph.osd.2.smithi067.stdout:Apr 22 22:32:24 smithi067 podman[54324]: 2024-04-22 22:32:24.714752752 +0000 UTC m=+0.648606278 container remove 5431738fc1b74708207e32fdd1bbbed0a2462c4840ddce2ea29bf2589d5d9501 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-2, ceph=True, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, release=754, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, RELEASE=HEAD, vcs-type=git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-22T22:32:25.017 INFO:journalctl@ceph.osd.2.smithi067.stdout:Apr 22 22:32:24 smithi067 podman[54401]: 2024-04-22 22:32:24.816336952 +0000 UTC m=+0.019548313 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-22T22:32:25.017 INFO:journalctl@ceph.osd.2.smithi067.stdout:Apr 22 22:32:24 smithi067 podman[54401]: 2024-04-22 22:32:24.932182167 +0000 UTC m=+0.135393519 container create 8ace6e46be8c6a3fd3dec8c25bbc7714ffe30dbadf88aa7698971af66372b084 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-2-deactivate, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, io.openshift.tags=base centos centos-stream, vcs-type=git, GIT_BRANCH=HEAD, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, ceph=True, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-22T22:32:25.443 INFO:journalctl@ceph.osd.2.smithi067.stdout:Apr 22 22:32:25 smithi067 podman[54401]: 2024-04-22 22:32:25.151108515 +0000 UTC m=+0.354319863 container init 8ace6e46be8c6a3fd3dec8c25bbc7714ffe30dbadf88aa7698971af66372b084 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, GIT_CLEAN=True, version=8, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., RELEASE=HEAD, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, ceph=True, release=754, name=centos-stream) 2024-04-22T22:32:25.443 INFO:journalctl@ceph.osd.2.smithi067.stdout:Apr 22 22:32:25 smithi067 podman[54401]: 2024-04-22 22:32:25.154260411 +0000 UTC m=+0.357471768 container start 8ace6e46be8c6a3fd3dec8c25bbc7714ffe30dbadf88aa7698971af66372b084 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, io.buildah.version=1.19.8, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, ceph=True, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc.) 2024-04-22T22:32:25.443 INFO:journalctl@ceph.osd.2.smithi067.stdout:Apr 22 22:32:25 smithi067 podman[54401]: 2024-04-22 22:32:25.225886924 +0000 UTC m=+0.429098281 container attach 8ace6e46be8c6a3fd3dec8c25bbc7714ffe30dbadf88aa7698971af66372b084 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-2-deactivate, version=8, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, vcs-type=git, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, release=754) 2024-04-22T22:32:25.444 INFO:journalctl@ceph.osd.2.smithi067.stdout:Apr 22 22:32:25 smithi067 podman[54401]: 2024-04-22 22:32:25.445247689 +0000 UTC m=+0.648459040 container died 8ace6e46be8c6a3fd3dec8c25bbc7714ffe30dbadf88aa7698971af66372b084 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-2-deactivate, vcs-type=git, ceph=True, architecture=x86_64, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, name=centos-stream, com.redhat.component=centos-stream-container, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, release=754) 2024-04-22T22:32:26.060 DEBUG:teuthology.orchestra.run.smithi067:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@osd.2.service' 2024-04-22T22:32:26.102 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T22:32:26.102 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-04-22T22:32:26.103 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-04-22T22:32:26.103 DEBUG:teuthology.orchestra.run.smithi067:> sudo systemctl stop ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@osd.3 2024-04-22T22:32:26.339 INFO:journalctl@ceph.osd.3.smithi067.stdout:Apr 22 22:32:26 smithi067 systemd[1]: Stopping Ceph osd.3 for 1ab1a3a8-00f6-11ef-bc93-c7b262605968... 2024-04-22T22:32:26.340 INFO:journalctl@ceph.osd.3.smithi067.stdout:Apr 22 22:32:26 smithi067 bash[54479]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd.3" found: no such container 2024-04-22T22:32:26.766 INFO:journalctl@ceph.osd.3.smithi067.stdout:Apr 22 22:32:26 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-3[38366]: 2024-04-22T22:32:26.340+0000 7fd8b5194700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T22:32:26.766 INFO:journalctl@ceph.osd.3.smithi067.stdout:Apr 22 22:32:26 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-3[38366]: 2024-04-22T22:32:26.340+0000 7fd8b5194700 -1 osd.3 95 *** Got signal Terminated *** 2024-04-22T22:32:26.767 INFO:journalctl@ceph.osd.3.smithi067.stdout:Apr 22 22:32:26 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-3[38366]: 2024-04-22T22:32:26.340+0000 7fd8b5194700 -1 osd.3 95 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T22:32:29.765 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:29 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:29.312+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:04.563374+0000 front 2024-04-22T22:32:04.563471+0000 (oldest deadline 2024-04-22T22:32:28.663454+0000) 2024-04-22T22:32:30.516 INFO:journalctl@ceph.osd.3.smithi067.stdout:Apr 22 22:32:30 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-3[38366]: 2024-04-22T22:32:30.210+0000 7fd8b0b2d700 -1 osd.3 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:06.007745+0000 front 2024-04-22T22:32:06.007827+0000 (oldest deadline 2024-04-22T22:32:30.107844+0000) 2024-04-22T22:32:30.765 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:30 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:30.266+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:04.563374+0000 front 2024-04-22T22:32:04.563471+0000 (oldest deadline 2024-04-22T22:32:28.663454+0000) 2024-04-22T22:32:31.516 INFO:journalctl@ceph.osd.3.smithi067.stdout:Apr 22 22:32:31 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-3[38366]: 2024-04-22T22:32:31.169+0000 7fd8b0b2d700 -1 osd.3 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:06.007745+0000 front 2024-04-22T22:32:06.007827+0000 (oldest deadline 2024-04-22T22:32:30.107844+0000) 2024-04-22T22:32:31.765 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:31 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:31.302+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:04.563374+0000 front 2024-04-22T22:32:04.563471+0000 (oldest deadline 2024-04-22T22:32:28.663454+0000) 2024-04-22T22:32:31.980 INFO:journalctl@ceph.osd.3.smithi067.stdout:Apr 22 22:32:31 smithi067 podman[54490]: 2024-04-22 22:32:31.706116785 +0000 UTC m=+5.487365156 container died ea1ecfb9c93118962ff8699a680073cd3b6d5eebc087a50321df4a024dd000a6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-3, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, version=8, ceph=True, io.openshift.expose-services=, vcs-type=git, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-22T22:32:32.266 INFO:journalctl@ceph.osd.3.smithi067.stdout:Apr 22 22:32:32 smithi067 podman[54490]: 2024-04-22 22:32:32.081604799 +0000 UTC m=+5.862853169 container cleanup ea1ecfb9c93118962ff8699a680073cd3b6d5eebc087a50321df4a024dd000a6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-3, io.openshift.tags=base centos centos-stream, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, io.buildah.version=1.19.8, vendor=Red Hat, Inc., version=8, GIT_CLEAN=True, io.openshift.expose-services=, RELEASE=HEAD, name=centos-stream, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, vcs-type=git, distribution-scope=public, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-22T22:32:32.267 INFO:journalctl@ceph.osd.3.smithi067.stdout:Apr 22 22:32:32 smithi067 bash[54490]: ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-3 2024-04-22T22:32:32.267 INFO:journalctl@ceph.osd.3.smithi067.stdout:Apr 22 22:32:32 smithi067 bash[54520]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd.3" found: no such container 2024-04-22T22:32:32.681 INFO:journalctl@ceph.osd.3.smithi067.stdout:Apr 22 22:32:32 smithi067 podman[54507]: 2024-04-22 22:32:32.366320124 +0000 UTC m=+0.655842505 container remove ea1ecfb9c93118962ff8699a680073cd3b6d5eebc087a50321df4a024dd000a6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-3, version=8, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, name=centos-stream, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, vcs-type=git, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, ceph=True, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-22T22:32:32.681 INFO:journalctl@ceph.osd.3.smithi067.stdout:Apr 22 22:32:32 smithi067 podman[54589]: 2024-04-22 22:32:32.484584568 +0000 UTC m=+0.020489076 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-22T22:32:32.681 INFO:journalctl@ceph.osd.3.smithi067.stdout:Apr 22 22:32:32 smithi067 podman[54589]: 2024-04-22 22:32:32.583647257 +0000 UTC m=+0.119551788 container create 80d124bb8258f12b198b131b3dcab21d42bea0edd178e47ca44640cd17a4c694 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-3-deactivate, io.openshift.expose-services=, name=centos-stream, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, version=8, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64) 2024-04-22T22:32:32.765 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:32 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:32.291+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:04.563374+0000 front 2024-04-22T22:32:04.563471+0000 (oldest deadline 2024-04-22T22:32:28.663454+0000) 2024-04-22T22:32:33.016 INFO:journalctl@ceph.osd.3.smithi067.stdout:Apr 22 22:32:32 smithi067 podman[54589]: 2024-04-22 22:32:32.786135148 +0000 UTC m=+0.322039636 container init 80d124bb8258f12b198b131b3dcab21d42bea0edd178e47ca44640cd17a4c694 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-3-deactivate, io.openshift.expose-services=, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, distribution-scope=public, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, maintainer=Guillaume Abrioux ) 2024-04-22T22:32:33.017 INFO:journalctl@ceph.osd.3.smithi067.stdout:Apr 22 22:32:32 smithi067 podman[54589]: 2024-04-22 22:32:32.789530261 +0000 UTC m=+0.325434750 container start 80d124bb8258f12b198b131b3dcab21d42bea0edd178e47ca44640cd17a4c694 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-3-deactivate, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , release=754, io.openshift.expose-services=, GIT_CLEAN=True, ceph=True, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, vcs-type=git, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, version=8, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-22T22:32:33.017 INFO:journalctl@ceph.osd.3.smithi067.stdout:Apr 22 22:32:32 smithi067 podman[54589]: 2024-04-22 22:32:32.886090783 +0000 UTC m=+0.421995281 container attach 80d124bb8258f12b198b131b3dcab21d42bea0edd178e47ca44640cd17a4c694 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-3-deactivate, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, RELEASE=HEAD, name=centos-stream, vendor=Red Hat, Inc., vcs-type=git, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, com.redhat.component=centos-stream-container, architecture=x86_64, release=754, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870) 2024-04-22T22:32:33.222 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:32 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-5[28891]: 2024-04-22T22:32:32.900+0000 7f63a663e700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.333477+0000 front 2024-04-22T22:32:07.333315+0000 (oldest deadline 2024-04-22T22:32:32.033401+0000) 2024-04-22T22:32:33.379 INFO:journalctl@ceph.osd.3.smithi067.stdout:Apr 22 22:32:33 smithi067 podman[54589]: 2024-04-22 22:32:33.083072981 +0000 UTC m=+0.618977470 container died 80d124bb8258f12b198b131b3dcab21d42bea0edd178e47ca44640cd17a4c694 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-3-deactivate, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, ceph=True, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, release=754, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, vcs-type=git, io.buildah.version=1.19.8) 2024-04-22T22:32:33.515 INFO:journalctl@ceph.osd.4.smithi086.stdout:Apr 22 22:32:33 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-4[25948]: 2024-04-22T22:32:33.223+0000 7f2615385700 -1 osd.4 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.820547+0000 front 2024-04-22T22:32:07.820578+0000 (oldest deadline 2024-04-22T22:32:33.120429+0000) 2024-04-22T22:32:33.515 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:33 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:33.308+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:04.563374+0000 front 2024-04-22T22:32:04.563471+0000 (oldest deadline 2024-04-22T22:32:28.663454+0000) 2024-04-22T22:32:33.631 INFO:journalctl@ceph.osd.3.smithi067.stdout:Apr 22 22:32:33 smithi067 podman[54615]: 2024-04-22 22:32:33.630891455 +0000 UTC m=+0.542079549 container remove 80d124bb8258f12b198b131b3dcab21d42bea0edd178e47ca44640cd17a4c694 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-3-deactivate, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, ceph=True, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, GIT_CLEAN=True, GIT_BRANCH=HEAD, vcs-type=git, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, distribution-scope=public, vendor=Red Hat, Inc., name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870) 2024-04-22T22:32:33.653 DEBUG:teuthology.orchestra.run.smithi067:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@osd.3.service' 2024-04-22T22:32:33.695 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T22:32:33.695 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-04-22T22:32:33.696 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-04-22T22:32:33.696 DEBUG:teuthology.orchestra.run.smithi086:> sudo systemctl stop ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@osd.4 2024-04-22T22:32:33.883 INFO:journalctl@ceph.osd.4.smithi086.stdout:Apr 22 22:32:33 smithi086 systemd[1]: Stopping Ceph osd.4 for 1ab1a3a8-00f6-11ef-bc93-c7b262605968... 2024-04-22T22:32:33.883 INFO:journalctl@ceph.osd.4.smithi086.stdout:Apr 22 22:32:33 smithi086 bash[43374]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd.4" found: no such container 2024-04-22T22:32:33.883 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:33 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:33.579+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.436533+0000 front 2024-04-22T22:32:07.436655+0000 (oldest deadline 2024-04-22T22:32:32.736485+0000) 2024-04-22T22:32:34.212 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:33 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-5[28891]: 2024-04-22T22:32:33.882+0000 7f63a663e700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.333477+0000 front 2024-04-22T22:32:07.333315+0000 (oldest deadline 2024-04-22T22:32:32.033401+0000) 2024-04-22T22:32:34.212 INFO:journalctl@ceph.osd.4.smithi086.stdout:Apr 22 22:32:33 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-4[25948]: 2024-04-22T22:32:33.886+0000 7f26191eb700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T22:32:34.213 INFO:journalctl@ceph.osd.4.smithi086.stdout:Apr 22 22:32:33 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-4[25948]: 2024-04-22T22:32:33.886+0000 7f26191eb700 -1 osd.4 95 *** Got signal Terminated *** 2024-04-22T22:32:34.213 INFO:journalctl@ceph.osd.4.smithi086.stdout:Apr 22 22:32:33 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-4[25948]: 2024-04-22T22:32:33.886+0000 7f26191eb700 -1 osd.4 95 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T22:32:34.213 INFO:journalctl@ceph.osd.4.smithi086.stdout:Apr 22 22:32:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-4[25948]: 2024-04-22T22:32:34.213+0000 7f2615385700 -1 osd.4 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.820547+0000 front 2024-04-22T22:32:07.820578+0000 (oldest deadline 2024-04-22T22:32:33.120429+0000) 2024-04-22T22:32:34.515 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:34.284+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:04.563374+0000 front 2024-04-22T22:32:04.563471+0000 (oldest deadline 2024-04-22T22:32:28.663454+0000) 2024-04-22T22:32:34.894 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:34.530+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.436533+0000 front 2024-04-22T22:32:07.436655+0000 (oldest deadline 2024-04-22T22:32:32.736485+0000) 2024-04-22T22:32:35.243 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:34 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-5[28891]: 2024-04-22T22:32:34.895+0000 7f63a663e700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.333477+0000 front 2024-04-22T22:32:07.333315+0000 (oldest deadline 2024-04-22T22:32:32.033401+0000) 2024-04-22T22:32:35.515 INFO:journalctl@ceph.osd.4.smithi086.stdout:Apr 22 22:32:35 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-4[25948]: 2024-04-22T22:32:35.255+0000 7f2615385700 -1 osd.4 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.820547+0000 front 2024-04-22T22:32:07.820578+0000 (oldest deadline 2024-04-22T22:32:33.120429+0000) 2024-04-22T22:32:35.516 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:35 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:35.243+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:04.563374+0000 front 2024-04-22T22:32:04.563471+0000 (oldest deadline 2024-04-22T22:32:28.663454+0000) 2024-04-22T22:32:35.861 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:35 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:35.553+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.436533+0000 front 2024-04-22T22:32:07.436655+0000 (oldest deadline 2024-04-22T22:32:32.736485+0000) 2024-04-22T22:32:36.209 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:35 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-5[28891]: 2024-04-22T22:32:35.862+0000 7f63a663e700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.333477+0000 front 2024-04-22T22:32:07.333315+0000 (oldest deadline 2024-04-22T22:32:32.033401+0000) 2024-04-22T22:32:36.515 INFO:journalctl@ceph.osd.4.smithi086.stdout:Apr 22 22:32:36 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-4[25948]: 2024-04-22T22:32:36.210+0000 7f2615385700 -1 osd.4 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.820547+0000 front 2024-04-22T22:32:07.820578+0000 (oldest deadline 2024-04-22T22:32:33.120429+0000) 2024-04-22T22:32:36.515 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:36 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:36.255+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:04.563374+0000 front 2024-04-22T22:32:04.563471+0000 (oldest deadline 2024-04-22T22:32:28.663454+0000) 2024-04-22T22:32:36.896 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:36 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:36.551+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.436533+0000 front 2024-04-22T22:32:07.436655+0000 (oldest deadline 2024-04-22T22:32:32.736485+0000) 2024-04-22T22:32:37.245 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:36 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-5[28891]: 2024-04-22T22:32:36.897+0000 7f63a663e700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.333477+0000 front 2024-04-22T22:32:07.333315+0000 (oldest deadline 2024-04-22T22:32:32.033401+0000) 2024-04-22T22:32:37.515 INFO:journalctl@ceph.osd.4.smithi086.stdout:Apr 22 22:32:37 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-4[25948]: 2024-04-22T22:32:37.245+0000 7f2615385700 -1 osd.4 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.820547+0000 front 2024-04-22T22:32:07.820578+0000 (oldest deadline 2024-04-22T22:32:33.120429+0000) 2024-04-22T22:32:37.515 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:37 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:37.288+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:04.563374+0000 front 2024-04-22T22:32:04.563471+0000 (oldest deadline 2024-04-22T22:32:28.663454+0000) 2024-04-22T22:32:37.895 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:37 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:37.548+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.436533+0000 front 2024-04-22T22:32:07.436655+0000 (oldest deadline 2024-04-22T22:32:32.736485+0000) 2024-04-22T22:32:38.221 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:37 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-5[28891]: 2024-04-22T22:32:37.897+0000 7f63a663e700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.333477+0000 front 2024-04-22T22:32:07.333315+0000 (oldest deadline 2024-04-22T22:32:32.033401+0000) 2024-04-22T22:32:38.222 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:37 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-5[28891]: 2024-04-22T22:32:37.897+0000 7f63a663e700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:12.033843+0000 front 2024-04-22T22:32:12.033764+0000 (oldest deadline 2024-04-22T22:32:37.333812+0000) 2024-04-22T22:32:38.515 INFO:journalctl@ceph.osd.4.smithi086.stdout:Apr 22 22:32:38 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-4[25948]: 2024-04-22T22:32:38.223+0000 7f2615385700 -1 osd.4 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.820547+0000 front 2024-04-22T22:32:07.820578+0000 (oldest deadline 2024-04-22T22:32:33.120429+0000) 2024-04-22T22:32:38.516 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:38 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:38.299+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:04.563374+0000 front 2024-04-22T22:32:04.563471+0000 (oldest deadline 2024-04-22T22:32:28.663454+0000) 2024-04-22T22:32:38.906 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:38 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:38.527+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.436533+0000 front 2024-04-22T22:32:07.436655+0000 (oldest deadline 2024-04-22T22:32:32.736485+0000) 2024-04-22T22:32:39.230 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:38 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-5[28891]: 2024-04-22T22:32:38.905+0000 7f63a663e700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.333477+0000 front 2024-04-22T22:32:07.333315+0000 (oldest deadline 2024-04-22T22:32:32.033401+0000) 2024-04-22T22:32:39.231 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:38 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-5[28891]: 2024-04-22T22:32:38.905+0000 7f63a663e700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:12.033843+0000 front 2024-04-22T22:32:12.033764+0000 (oldest deadline 2024-04-22T22:32:37.333812+0000) 2024-04-22T22:32:39.503 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:39.332+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:04.563374+0000 front 2024-04-22T22:32:04.563471+0000 (oldest deadline 2024-04-22T22:32:28.663454+0000) 2024-04-22T22:32:39.503 INFO:journalctl@ceph.osd.4.smithi086.stdout:Apr 22 22:32:39 smithi086 podman[43384]: 2024-04-22 22:32:39.232328957 +0000 UTC m=+5.445571567 container died 4196c45a856b0a67c7baa6b955a95ef86172c0c6777c6baf7808074f81ae9107 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-4, name=centos-stream, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, release=754, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870) 2024-04-22T22:32:39.765 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:39.558+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.436533+0000 front 2024-04-22T22:32:07.436655+0000 (oldest deadline 2024-04-22T22:32:32.736485+0000) 2024-04-22T22:32:39.766 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:39.558+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:12.736922+0000 front 2024-04-22T22:32:12.736974+0000 (oldest deadline 2024-04-22T22:32:38.636942+0000) 2024-04-22T22:32:39.766 INFO:journalctl@ceph.osd.4.smithi086.stdout:Apr 22 22:32:39 smithi086 podman[43384]: 2024-04-22 22:32:39.612568768 +0000 UTC m=+5.825811389 container cleanup 4196c45a856b0a67c7baa6b955a95ef86172c0c6777c6baf7808074f81ae9107 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-4, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, ceph=True, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , distribution-scope=public, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, version=8, GIT_CLEAN=True, release=754, architecture=x86_64) 2024-04-22T22:32:39.766 INFO:journalctl@ceph.osd.4.smithi086.stdout:Apr 22 22:32:39 smithi086 bash[43384]: ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-4 2024-04-22T22:32:39.766 INFO:journalctl@ceph.osd.4.smithi086.stdout:Apr 22 22:32:39 smithi086 bash[43414]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd.4" found: no such container 2024-04-22T22:32:40.159 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-5[28891]: 2024-04-22T22:32:39.881+0000 7f63a663e700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.333477+0000 front 2024-04-22T22:32:07.333315+0000 (oldest deadline 2024-04-22T22:32:32.033401+0000) 2024-04-22T22:32:40.160 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:39 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-5[28891]: 2024-04-22T22:32:39.881+0000 7f63a663e700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:12.033843+0000 front 2024-04-22T22:32:12.033764+0000 (oldest deadline 2024-04-22T22:32:37.333812+0000) 2024-04-22T22:32:40.160 INFO:journalctl@ceph.osd.4.smithi086.stdout:Apr 22 22:32:39 smithi086 podman[43401]: 2024-04-22 22:32:39.872159356 +0000 UTC m=+0.637990468 container remove 4196c45a856b0a67c7baa6b955a95ef86172c0c6777c6baf7808074f81ae9107 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-4, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., version=8, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , ceph=True, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, vcs-type=git) 2024-04-22T22:32:40.160 INFO:journalctl@ceph.osd.4.smithi086.stdout:Apr 22 22:32:40 smithi086 podman[43478]: 2024-04-22 22:32:39.951695821 +0000 UTC m=+0.019264787 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-22T22:32:40.160 INFO:journalctl@ceph.osd.4.smithi086.stdout:Apr 22 22:32:40 smithi086 podman[43478]: 2024-04-22 22:32:40.064804802 +0000 UTC m=+0.132373750 container create 5d6b699b1895469a13f5a42f12f5c6be378f647a2d23976a8197e5b93515ee65 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-4-deactivate, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, architecture=x86_64, version=8, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0) 2024-04-22T22:32:40.515 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:40 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:40.307+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:04.563374+0000 front 2024-04-22T22:32:04.563471+0000 (oldest deadline 2024-04-22T22:32:28.663454+0000) 2024-04-22T22:32:40.515 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:40 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:40.307+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:14.365427+0000 front 2024-04-22T22:32:14.365553+0000 (oldest deadline 2024-04-22T22:32:40.265418+0000) 2024-04-22T22:32:40.516 INFO:journalctl@ceph.osd.4.smithi086.stdout:Apr 22 22:32:40 smithi086 podman[43478]: 2024-04-22 22:32:40.258666625 +0000 UTC m=+0.326235573 container init 5d6b699b1895469a13f5a42f12f5c6be378f647a2d23976a8197e5b93515ee65 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-4-deactivate, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, architecture=x86_64, name=centos-stream, release=754, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, vendor=Red Hat, Inc., vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream) 2024-04-22T22:32:40.516 INFO:journalctl@ceph.osd.4.smithi086.stdout:Apr 22 22:32:40 smithi086 podman[43478]: 2024-04-22 22:32:40.261827349 +0000 UTC m=+0.329396300 container start 5d6b699b1895469a13f5a42f12f5c6be378f647a2d23976a8197e5b93515ee65 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-4-deactivate, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, GIT_CLEAN=True, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, version=8, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-22T22:32:40.516 INFO:journalctl@ceph.osd.4.smithi086.stdout:Apr 22 22:32:40 smithi086 podman[43478]: 2024-04-22 22:32:40.333565339 +0000 UTC m=+0.401134290 container attach 5d6b699b1895469a13f5a42f12f5c6be378f647a2d23976a8197e5b93515ee65 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-4-deactivate, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, release=754, io.buildah.version=1.19.8, RELEASE=HEAD, GIT_CLEAN=True, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, io.openshift.tags=base centos centos-stream, architecture=x86_64, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-22T22:32:40.818 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:40 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:40.519+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.436533+0000 front 2024-04-22T22:32:07.436655+0000 (oldest deadline 2024-04-22T22:32:32.736485+0000) 2024-04-22T22:32:40.818 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:40 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:40.519+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:12.736922+0000 front 2024-04-22T22:32:12.736974+0000 (oldest deadline 2024-04-22T22:32:38.636942+0000) 2024-04-22T22:32:40.819 INFO:journalctl@ceph.osd.4.smithi086.stdout:Apr 22 22:32:40 smithi086 podman[43478]: 2024-04-22 22:32:40.540525961 +0000 UTC m=+0.608094917 container died 5d6b699b1895469a13f5a42f12f5c6be378f647a2d23976a8197e5b93515ee65 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-4-deactivate, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, architecture=x86_64, ceph=True, vcs-type=git, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, RELEASE=HEAD, maintainer=Guillaume Abrioux , io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., name=centos-stream, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-22T22:32:41.068 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:40 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-5[28891]: 2024-04-22T22:32:40.865+0000 7f63a663e700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.333477+0000 front 2024-04-22T22:32:07.333315+0000 (oldest deadline 2024-04-22T22:32:32.033401+0000) 2024-04-22T22:32:41.069 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:40 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-5[28891]: 2024-04-22T22:32:40.865+0000 7f63a663e700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:12.033843+0000 front 2024-04-22T22:32:12.033764+0000 (oldest deadline 2024-04-22T22:32:37.333812+0000) 2024-04-22T22:32:41.069 INFO:journalctl@ceph.osd.4.smithi086.stdout:Apr 22 22:32:41 smithi086 podman[43504]: 2024-04-22 22:32:41.053854346 +0000 UTC m=+0.508328092 container remove 5d6b699b1895469a13f5a42f12f5c6be378f647a2d23976a8197e5b93515ee65 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-4-deactivate, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, name=centos-stream, architecture=x86_64, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, RELEASE=HEAD, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , release=754) 2024-04-22T22:32:41.069 INFO:journalctl@ceph.osd.4.smithi086.stdout:Apr 22 22:32:41 smithi086 systemd[1]: ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@osd.4.service: Deactivated successfully. 2024-04-22T22:32:41.072 DEBUG:teuthology.orchestra.run.smithi086:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@osd.4.service' 2024-04-22T22:32:41.112 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T22:32:41.112 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-04-22T22:32:41.113 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-04-22T22:32:41.113 DEBUG:teuthology.orchestra.run.smithi086:> sudo systemctl stop ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@osd.5 2024-04-22T22:32:41.343 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:41 smithi086 systemd[1]: Stopping Ceph osd.5 for 1ab1a3a8-00f6-11ef-bc93-c7b262605968... 2024-04-22T22:32:41.343 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:41 smithi086 bash[43557]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd.5" found: no such container 2024-04-22T22:32:41.343 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:41 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-5[28891]: 2024-04-22T22:32:41.314+0000 7f63a8ca1700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T22:32:41.343 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:41 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-5[28891]: 2024-04-22T22:32:41.314+0000 7f63a8ca1700 -1 osd.5 95 *** Got signal Terminated *** 2024-04-22T22:32:41.343 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:41 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-5[28891]: 2024-04-22T22:32:41.314+0000 7f63a8ca1700 -1 osd.5 95 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T22:32:41.765 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:41 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:41.529+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.436533+0000 front 2024-04-22T22:32:07.436655+0000 (oldest deadline 2024-04-22T22:32:32.736485+0000) 2024-04-22T22:32:41.765 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:41 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:41.529+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:12.736922+0000 front 2024-04-22T22:32:12.736974+0000 (oldest deadline 2024-04-22T22:32:38.636942+0000) 2024-04-22T22:32:41.766 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:41 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:41.343+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:04.563374+0000 front 2024-04-22T22:32:04.563471+0000 (oldest deadline 2024-04-22T22:32:28.663454+0000) 2024-04-22T22:32:41.766 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:41 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:41.343+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:14.365427+0000 front 2024-04-22T22:32:14.365553+0000 (oldest deadline 2024-04-22T22:32:40.265418+0000) 2024-04-22T22:32:42.209 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:41 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-5[28891]: 2024-04-22T22:32:41.913+0000 7f63a663e700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.333477+0000 front 2024-04-22T22:32:07.333315+0000 (oldest deadline 2024-04-22T22:32:32.033401+0000) 2024-04-22T22:32:42.209 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:41 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-5[28891]: 2024-04-22T22:32:41.913+0000 7f63a663e700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:12.033843+0000 front 2024-04-22T22:32:12.033764+0000 (oldest deadline 2024-04-22T22:32:37.333812+0000) 2024-04-22T22:32:42.515 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:42 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:42.352+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:04.563374+0000 front 2024-04-22T22:32:04.563471+0000 (oldest deadline 2024-04-22T22:32:28.663454+0000) 2024-04-22T22:32:42.515 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:42 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:42.352+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:14.365427+0000 front 2024-04-22T22:32:14.365553+0000 (oldest deadline 2024-04-22T22:32:40.265418+0000) 2024-04-22T22:32:42.875 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:42 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:42.550+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.436533+0000 front 2024-04-22T22:32:07.436655+0000 (oldest deadline 2024-04-22T22:32:32.736485+0000) 2024-04-22T22:32:42.875 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:42 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:42.550+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:12.736922+0000 front 2024-04-22T22:32:12.736974+0000 (oldest deadline 2024-04-22T22:32:38.636942+0000) 2024-04-22T22:32:43.221 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:42 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-5[28891]: 2024-04-22T22:32:42.872+0000 7f63a663e700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.333477+0000 front 2024-04-22T22:32:07.333315+0000 (oldest deadline 2024-04-22T22:32:32.033401+0000) 2024-04-22T22:32:43.221 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:42 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-5[28891]: 2024-04-22T22:32:42.872+0000 7f63a663e700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:12.033843+0000 front 2024-04-22T22:32:12.033764+0000 (oldest deadline 2024-04-22T22:32:37.333812+0000) 2024-04-22T22:32:43.515 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:43 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:43.334+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:04.563374+0000 front 2024-04-22T22:32:04.563471+0000 (oldest deadline 2024-04-22T22:32:28.663454+0000) 2024-04-22T22:32:43.515 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:43 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:43.334+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:14.365427+0000 front 2024-04-22T22:32:14.365553+0000 (oldest deadline 2024-04-22T22:32:40.265418+0000) 2024-04-22T22:32:43.824 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:43 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:43.544+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.436533+0000 front 2024-04-22T22:32:07.436655+0000 (oldest deadline 2024-04-22T22:32:32.736485+0000) 2024-04-22T22:32:43.824 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:43 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:43.544+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:12.736922+0000 front 2024-04-22T22:32:12.736974+0000 (oldest deadline 2024-04-22T22:32:38.636942+0000) 2024-04-22T22:32:44.265 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:43 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-5[28891]: 2024-04-22T22:32:43.824+0000 7f63a663e700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.333477+0000 front 2024-04-22T22:32:07.333315+0000 (oldest deadline 2024-04-22T22:32:32.033401+0000) 2024-04-22T22:32:44.265 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:43 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-5[28891]: 2024-04-22T22:32:43.824+0000 7f63a663e700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:12.033843+0000 front 2024-04-22T22:32:12.033764+0000 (oldest deadline 2024-04-22T22:32:37.333812+0000) 2024-04-22T22:32:44.765 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:44 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:44.525+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.436533+0000 front 2024-04-22T22:32:07.436655+0000 (oldest deadline 2024-04-22T22:32:32.736485+0000) 2024-04-22T22:32:44.765 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:44 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:44.525+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:12.736922+0000 front 2024-04-22T22:32:12.736974+0000 (oldest deadline 2024-04-22T22:32:38.636942+0000) 2024-04-22T22:32:44.766 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:44 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:44.301+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:04.563374+0000 front 2024-04-22T22:32:04.563471+0000 (oldest deadline 2024-04-22T22:32:28.663454+0000) 2024-04-22T22:32:44.766 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:44 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:44.301+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:14.365427+0000 front 2024-04-22T22:32:14.365553+0000 (oldest deadline 2024-04-22T22:32:40.265418+0000) 2024-04-22T22:32:45.265 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:44 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-5[28891]: 2024-04-22T22:32:44.793+0000 7f63a663e700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.333477+0000 front 2024-04-22T22:32:07.333315+0000 (oldest deadline 2024-04-22T22:32:32.033401+0000) 2024-04-22T22:32:45.265 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:44 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-5[28891]: 2024-04-22T22:32:44.793+0000 7f63a663e700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:12.033843+0000 front 2024-04-22T22:32:12.033764+0000 (oldest deadline 2024-04-22T22:32:37.333812+0000) 2024-04-22T22:32:45.265 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:44 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-5[28891]: 2024-04-22T22:32:44.793+0000 7f63a663e700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.67:6822 osd.2 since back 2024-04-22T22:32:19.634885+0000 front 2024-04-22T22:32:19.634906+0000 (oldest deadline 2024-04-22T22:32:44.235374+0000) 2024-04-22T22:32:45.569 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:45 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:45.295+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:04.563374+0000 front 2024-04-22T22:32:04.563471+0000 (oldest deadline 2024-04-22T22:32:28.663454+0000) 2024-04-22T22:32:45.569 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:45 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:45.295+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:14.365427+0000 front 2024-04-22T22:32:14.365553+0000 (oldest deadline 2024-04-22T22:32:40.265418+0000) 2024-04-22T22:32:45.570 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:45 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:45.295+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6822 osd.2 since back 2024-04-22T22:32:20.266121+0000 front 2024-04-22T22:32:20.266059+0000 (oldest deadline 2024-04-22T22:32:44.365892+0000) 2024-04-22T22:32:46.015 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:45 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:45.570+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.436533+0000 front 2024-04-22T22:32:07.436655+0000 (oldest deadline 2024-04-22T22:32:32.736485+0000) 2024-04-22T22:32:46.016 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:45 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:45.570+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:12.736922+0000 front 2024-04-22T22:32:12.736974+0000 (oldest deadline 2024-04-22T22:32:38.636942+0000) 2024-04-22T22:32:46.016 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:45 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-5[28891]: 2024-04-22T22:32:45.775+0000 7f63a663e700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.333477+0000 front 2024-04-22T22:32:07.333315+0000 (oldest deadline 2024-04-22T22:32:32.033401+0000) 2024-04-22T22:32:46.016 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:45 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-5[28891]: 2024-04-22T22:32:45.775+0000 7f63a663e700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:12.033843+0000 front 2024-04-22T22:32:12.033764+0000 (oldest deadline 2024-04-22T22:32:37.333812+0000) 2024-04-22T22:32:46.016 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:45 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-5[28891]: 2024-04-22T22:32:45.775+0000 7f63a663e700 -1 osd.5 95 heartbeat_check: no reply from 172.21.15.67:6822 osd.2 since back 2024-04-22T22:32:19.634885+0000 front 2024-04-22T22:32:19.634906+0000 (oldest deadline 2024-04-22T22:32:44.235374+0000) 2024-04-22T22:32:46.574 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:46 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:46.302+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:04.563374+0000 front 2024-04-22T22:32:04.563471+0000 (oldest deadline 2024-04-22T22:32:28.663454+0000) 2024-04-22T22:32:46.574 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:46 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:46.302+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:14.365427+0000 front 2024-04-22T22:32:14.365553+0000 (oldest deadline 2024-04-22T22:32:40.265418+0000) 2024-04-22T22:32:46.574 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:46 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:46.302+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6822 osd.2 since back 2024-04-22T22:32:20.266121+0000 front 2024-04-22T22:32:20.266059+0000 (oldest deadline 2024-04-22T22:32:44.365892+0000) 2024-04-22T22:32:46.956 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:46 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:46.575+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.436533+0000 front 2024-04-22T22:32:07.436655+0000 (oldest deadline 2024-04-22T22:32:32.736485+0000) 2024-04-22T22:32:46.957 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:46 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:46.575+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:12.736922+0000 front 2024-04-22T22:32:12.736974+0000 (oldest deadline 2024-04-22T22:32:38.636942+0000) 2024-04-22T22:32:46.957 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:46 smithi086 podman[43570]: 2024-04-22 22:32:46.708251305 +0000 UTC m=+5.481492335 container died 6c685d1d93a32ac5377f6c9cdd66912ad46ceaad89e1982ab0cad24d2d653a4b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-5, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, GIT_CLEAN=True, release=754, GIT_BRANCH=HEAD, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-22T22:32:47.248 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:47 smithi086 podman[43570]: 2024-04-22 22:32:47.048564864 +0000 UTC m=+5.821805887 container cleanup 6c685d1d93a32ac5377f6c9cdd66912ad46ceaad89e1982ab0cad24d2d653a4b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-5, io.k8s.display-name=CentOS Stream 8, vcs-type=git, GIT_CLEAN=True, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, release=754, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, vendor=Red Hat, Inc., architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , distribution-scope=public, io.buildah.version=1.19.8) 2024-04-22T22:32:47.249 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:47 smithi086 bash[43570]: ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-5 2024-04-22T22:32:47.515 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:47 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:47.323+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:04.563374+0000 front 2024-04-22T22:32:04.563471+0000 (oldest deadline 2024-04-22T22:32:28.663454+0000) 2024-04-22T22:32:47.515 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:47 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:47.323+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:14.365427+0000 front 2024-04-22T22:32:14.365553+0000 (oldest deadline 2024-04-22T22:32:40.265418+0000) 2024-04-22T22:32:47.515 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:47 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:47.323+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6822 osd.2 since back 2024-04-22T22:32:20.266121+0000 front 2024-04-22T22:32:20.266059+0000 (oldest deadline 2024-04-22T22:32:44.365892+0000) 2024-04-22T22:32:47.516 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:47 smithi086 podman[43600]: 2024-04-22 22:32:47.250352959 +0000 UTC m=+0.535605282 container remove 6c685d1d93a32ac5377f6c9cdd66912ad46ceaad89e1982ab0cad24d2d653a4b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-5, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, vcs-type=git, build-date=2022-05-03T08:36:31.336870, version=8, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, ceph=True, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., name=centos-stream, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, RELEASE=HEAD, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD) 2024-04-22T22:32:47.516 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:47 smithi086 bash[43614]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd.5" found: no such container 2024-04-22T22:32:47.516 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:47 smithi086 podman[43679]: 2024-04-22 22:32:47.399297731 +0000 UTC m=+0.019433371 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-22T22:32:47.516 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:47 smithi086 podman[43679]: 2024-04-22 22:32:47.500208006 +0000 UTC m=+0.120343646 container create f50bf63afe5757511f32c6e27243424f9f85f38ce5e47ce0f5a7f6b92f782ce4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-5-deactivate, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , architecture=x86_64, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, release=754, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, io.buildah.version=1.19.8, name=centos-stream, vcs-type=git) 2024-04-22T22:32:47.810 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:47 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:47.542+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.436533+0000 front 2024-04-22T22:32:07.436655+0000 (oldest deadline 2024-04-22T22:32:32.736485+0000) 2024-04-22T22:32:47.810 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:47 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:47.542+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:12.736922+0000 front 2024-04-22T22:32:12.736974+0000 (oldest deadline 2024-04-22T22:32:38.636942+0000) 2024-04-22T22:32:47.810 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:47 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:47.542+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6822 osd.2 since back 2024-04-22T22:32:22.538952+0000 front 2024-04-22T22:32:22.538878+0000 (oldest deadline 2024-04-22T22:32:47.238709+0000) 2024-04-22T22:32:47.810 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:47 smithi086 podman[43679]: 2024-04-22 22:32:47.711719471 +0000 UTC m=+0.331855110 container init f50bf63afe5757511f32c6e27243424f9f85f38ce5e47ce0f5a7f6b92f782ce4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-5-deactivate, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, io.buildah.version=1.19.8, ceph=True, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, GIT_CLEAN=True, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, vcs-type=git, distribution-scope=public, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, release=754) 2024-04-22T22:32:47.811 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:47 smithi086 podman[43679]: 2024-04-22 22:32:47.714856339 +0000 UTC m=+0.334991983 container start f50bf63afe5757511f32c6e27243424f9f85f38ce5e47ce0f5a7f6b92f782ce4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-5-deactivate, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., release=754, name=centos-stream, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, ceph=True, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, distribution-scope=public, GIT_CLEAN=True, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=) 2024-04-22T22:32:47.811 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:47 smithi086 podman[43679]: 2024-04-22 22:32:47.811959193 +0000 UTC m=+0.432094835 container attach f50bf63afe5757511f32c6e27243424f9f85f38ce5e47ce0f5a7f6b92f782ce4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-5-deactivate, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, name=centos-stream, release=754, distribution-scope=public, GIT_CLEAN=True, ceph=True, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-22T22:32:48.265 INFO:journalctl@ceph.osd.5.smithi086.stdout:Apr 22 22:32:47 smithi086 podman[43679]: 2024-04-22 22:32:47.994160637 +0000 UTC m=+0.614296276 container died f50bf63afe5757511f32c6e27243424f9f85f38ce5e47ce0f5a7f6b92f782ce4 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., ceph=True, distribution-scope=public, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, version=8, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, release=754, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-22T22:32:48.532 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:48 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:48.342+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:04.563374+0000 front 2024-04-22T22:32:04.563471+0000 (oldest deadline 2024-04-22T22:32:28.663454+0000) 2024-04-22T22:32:48.533 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:48 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:48.342+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:14.365427+0000 front 2024-04-22T22:32:14.365553+0000 (oldest deadline 2024-04-22T22:32:40.265418+0000) 2024-04-22T22:32:48.533 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:48 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:48.342+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6822 osd.2 since back 2024-04-22T22:32:20.266121+0000 front 2024-04-22T22:32:20.266059+0000 (oldest deadline 2024-04-22T22:32:44.365892+0000) 2024-04-22T22:32:48.556 DEBUG:teuthology.orchestra.run.smithi086:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@osd.5.service' 2024-04-22T22:32:48.598 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T22:32:48.598 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-04-22T22:32:48.598 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-04-22T22:32:48.598 DEBUG:teuthology.orchestra.run.smithi086:> sudo systemctl stop ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@osd.6 2024-04-22T22:32:48.794 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:48 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:48.548+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.436533+0000 front 2024-04-22T22:32:07.436655+0000 (oldest deadline 2024-04-22T22:32:32.736485+0000) 2024-04-22T22:32:48.795 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:48 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:48.548+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:12.736922+0000 front 2024-04-22T22:32:12.736974+0000 (oldest deadline 2024-04-22T22:32:38.636942+0000) 2024-04-22T22:32:48.795 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:48 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:48.548+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6822 osd.2 since back 2024-04-22T22:32:22.538952+0000 front 2024-04-22T22:32:22.538878+0000 (oldest deadline 2024-04-22T22:32:47.238709+0000) 2024-04-22T22:32:48.795 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:48 smithi086 systemd[1]: Stopping Ceph osd.6 for 1ab1a3a8-00f6-11ef-bc93-c7b262605968... 2024-04-22T22:32:48.795 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:48 smithi086 bash[43757]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd.6" found: no such container 2024-04-22T22:32:48.795 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:48 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:48.795+0000 7fcfb62d3700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T22:32:48.795 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:48 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:48.795+0000 7fcfb62d3700 -1 osd.6 95 *** Got signal Terminated *** 2024-04-22T22:32:48.795 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:48 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:48.795+0000 7fcfb62d3700 -1 osd.6 95 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T22:32:49.766 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:49 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:49.360+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:04.563374+0000 front 2024-04-22T22:32:04.563471+0000 (oldest deadline 2024-04-22T22:32:28.663454+0000) 2024-04-22T22:32:49.767 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:49 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:49.360+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:14.365427+0000 front 2024-04-22T22:32:14.365553+0000 (oldest deadline 2024-04-22T22:32:40.265418+0000) 2024-04-22T22:32:49.767 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:49 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:49.360+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6822 osd.2 since back 2024-04-22T22:32:20.266121+0000 front 2024-04-22T22:32:20.266059+0000 (oldest deadline 2024-04-22T22:32:44.365892+0000) 2024-04-22T22:32:49.767 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:49 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:49.590+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.436533+0000 front 2024-04-22T22:32:07.436655+0000 (oldest deadline 2024-04-22T22:32:32.736485+0000) 2024-04-22T22:32:49.767 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:49 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:49.590+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:12.736922+0000 front 2024-04-22T22:32:12.736974+0000 (oldest deadline 2024-04-22T22:32:38.636942+0000) 2024-04-22T22:32:49.767 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:49 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:49.590+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6822 osd.2 since back 2024-04-22T22:32:22.538952+0000 front 2024-04-22T22:32:22.538878+0000 (oldest deadline 2024-04-22T22:32:47.238709+0000) 2024-04-22T22:32:50.765 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:50 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:50.541+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.436533+0000 front 2024-04-22T22:32:07.436655+0000 (oldest deadline 2024-04-22T22:32:32.736485+0000) 2024-04-22T22:32:50.766 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:50 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:50.541+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:12.736922+0000 front 2024-04-22T22:32:12.736974+0000 (oldest deadline 2024-04-22T22:32:38.636942+0000) 2024-04-22T22:32:50.766 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:50 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:50.541+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6822 osd.2 since back 2024-04-22T22:32:22.538952+0000 front 2024-04-22T22:32:22.538878+0000 (oldest deadline 2024-04-22T22:32:47.238709+0000) 2024-04-22T22:32:50.766 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:50 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:50.354+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:04.563374+0000 front 2024-04-22T22:32:04.563471+0000 (oldest deadline 2024-04-22T22:32:28.663454+0000) 2024-04-22T22:32:50.766 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:50 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:50.354+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:14.365427+0000 front 2024-04-22T22:32:14.365553+0000 (oldest deadline 2024-04-22T22:32:40.265418+0000) 2024-04-22T22:32:50.766 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:50 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:50.354+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6822 osd.2 since back 2024-04-22T22:32:20.266121+0000 front 2024-04-22T22:32:20.266059+0000 (oldest deadline 2024-04-22T22:32:44.365892+0000) 2024-04-22T22:32:51.765 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:51 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:51.511+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.436533+0000 front 2024-04-22T22:32:07.436655+0000 (oldest deadline 2024-04-22T22:32:32.736485+0000) 2024-04-22T22:32:51.766 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:51 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:51.511+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:12.736922+0000 front 2024-04-22T22:32:12.736974+0000 (oldest deadline 2024-04-22T22:32:38.636942+0000) 2024-04-22T22:32:51.766 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:51 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:51.511+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6822 osd.2 since back 2024-04-22T22:32:22.538952+0000 front 2024-04-22T22:32:22.538878+0000 (oldest deadline 2024-04-22T22:32:47.238709+0000) 2024-04-22T22:32:51.767 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:51 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:51.317+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:04.563374+0000 front 2024-04-22T22:32:04.563471+0000 (oldest deadline 2024-04-22T22:32:28.663454+0000) 2024-04-22T22:32:51.767 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:51 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:51.317+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:14.365427+0000 front 2024-04-22T22:32:14.365553+0000 (oldest deadline 2024-04-22T22:32:40.265418+0000) 2024-04-22T22:32:51.767 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:51 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:51.317+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6822 osd.2 since back 2024-04-22T22:32:20.266121+0000 front 2024-04-22T22:32:20.266059+0000 (oldest deadline 2024-04-22T22:32:44.365892+0000) 2024-04-22T22:32:52.765 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:52 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:52.292+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:04.563374+0000 front 2024-04-22T22:32:04.563471+0000 (oldest deadline 2024-04-22T22:32:28.663454+0000) 2024-04-22T22:32:52.766 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:52 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:52.292+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:14.365427+0000 front 2024-04-22T22:32:14.365553+0000 (oldest deadline 2024-04-22T22:32:40.265418+0000) 2024-04-22T22:32:52.766 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:52 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:52.292+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6822 osd.2 since back 2024-04-22T22:32:20.266121+0000 front 2024-04-22T22:32:20.266059+0000 (oldest deadline 2024-04-22T22:32:44.365892+0000) 2024-04-22T22:32:52.766 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:52 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:52.490+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.436533+0000 front 2024-04-22T22:32:07.436655+0000 (oldest deadline 2024-04-22T22:32:32.736485+0000) 2024-04-22T22:32:52.766 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:52 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:52.490+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:12.736922+0000 front 2024-04-22T22:32:12.736974+0000 (oldest deadline 2024-04-22T22:32:38.636942+0000) 2024-04-22T22:32:52.767 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:52 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:52.490+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6822 osd.2 since back 2024-04-22T22:32:22.538952+0000 front 2024-04-22T22:32:22.538878+0000 (oldest deadline 2024-04-22T22:32:47.238709+0000) 2024-04-22T22:32:52.767 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:52 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:52.490+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6830 osd.3 since back 2024-04-22T22:32:31.239839+0000 front 2024-04-22T22:32:31.239769+0000 (oldest deadline 2024-04-22T22:32:52.339823+0000) 2024-04-22T22:32:53.515 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:53 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:53.281+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:04.563374+0000 front 2024-04-22T22:32:04.563471+0000 (oldest deadline 2024-04-22T22:32:28.663454+0000) 2024-04-22T22:32:53.516 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:53 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:53.281+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:14.365427+0000 front 2024-04-22T22:32:14.365553+0000 (oldest deadline 2024-04-22T22:32:40.265418+0000) 2024-04-22T22:32:53.516 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:53 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:53.281+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6822 osd.2 since back 2024-04-22T22:32:20.266121+0000 front 2024-04-22T22:32:20.266059+0000 (oldest deadline 2024-04-22T22:32:44.365892+0000) 2024-04-22T22:32:53.516 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:53 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:53.474+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:07.436533+0000 front 2024-04-22T22:32:07.436655+0000 (oldest deadline 2024-04-22T22:32:32.736485+0000) 2024-04-22T22:32:53.516 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:53 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:53.474+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:12.736922+0000 front 2024-04-22T22:32:12.736974+0000 (oldest deadline 2024-04-22T22:32:38.636942+0000) 2024-04-22T22:32:53.516 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:53 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:53.474+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6822 osd.2 since back 2024-04-22T22:32:22.538952+0000 front 2024-04-22T22:32:22.538878+0000 (oldest deadline 2024-04-22T22:32:47.238709+0000) 2024-04-22T22:32:53.516 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:53 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6[31789]: 2024-04-22T22:32:53.474+0000 7fcfb1c6c700 -1 osd.6 95 heartbeat_check: no reply from 172.21.15.67:6830 osd.3 since back 2024-04-22T22:32:31.239839+0000 front 2024-04-22T22:32:31.239769+0000 (oldest deadline 2024-04-22T22:32:52.339823+0000) 2024-04-22T22:32:54.536 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:54 smithi086 podman[43770]: 2024-04-22 22:32:54.280663748 +0000 UTC m=+5.570402012 container died 7e586ba77fac6858389c4d8dff0e5ef00e854a87e768de07d313d801fe36eb22 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, version=8, ceph=True, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-22T22:32:54.537 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:54 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:54.294+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:04.563374+0000 front 2024-04-22T22:32:04.563471+0000 (oldest deadline 2024-04-22T22:32:28.663454+0000) 2024-04-22T22:32:54.537 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:54 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:54.295+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:14.365427+0000 front 2024-04-22T22:32:14.365553+0000 (oldest deadline 2024-04-22T22:32:40.265418+0000) 2024-04-22T22:32:54.537 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:54 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:54.295+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6822 osd.2 since back 2024-04-22T22:32:20.266121+0000 front 2024-04-22T22:32:20.266059+0000 (oldest deadline 2024-04-22T22:32:44.365892+0000) 2024-04-22T22:32:54.537 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:54 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:54.295+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6830 osd.3 since back 2024-04-22T22:32:30.767548+0000 front 2024-04-22T22:32:30.767529+0000 (oldest deadline 2024-04-22T22:32:53.667419+0000) 2024-04-22T22:32:54.804 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:54 smithi086 podman[43770]: 2024-04-22 22:32:54.595642915 +0000 UTC m=+5.885381164 container cleanup 7e586ba77fac6858389c4d8dff0e5ef00e854a87e768de07d313d801fe36eb22 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, io.openshift.expose-services=, architecture=x86_64, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , distribution-scope=public, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, name=centos-stream, version=8) 2024-04-22T22:32:54.804 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:54 smithi086 bash[43770]: ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6 2024-04-22T22:32:54.804 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:54 smithi086 bash[43800]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd.6" found: no such container 2024-04-22T22:32:55.129 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:54 smithi086 podman[43787]: 2024-04-22 22:32:54.805537596 +0000 UTC m=+0.520878093 container remove 7e586ba77fac6858389c4d8dff0e5ef00e854a87e768de07d313d801fe36eb22 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, ceph=True, RELEASE=HEAD, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, release=754, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, version=8) 2024-04-22T22:32:55.130 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:55 smithi086 podman[43867]: 2024-04-22 22:32:54.930533476 +0000 UTC m=+0.018523729 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-22T22:32:55.130 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:55 smithi086 podman[43867]: 2024-04-22 22:32:55.033391352 +0000 UTC m=+0.121381578 container create dd27e9fd63b78edc79aa18311d1ce2bb4e5a3d59899496c55dbd374360ecc9a2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6-deactivate, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., io.openshift.expose-services=, maintainer=Guillaume Abrioux , architecture=x86_64, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, GIT_CLEAN=True, version=8, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git) 2024-04-22T22:32:55.516 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:55 smithi086 podman[43867]: 2024-04-22 22:32:55.244206251 +0000 UTC m=+0.332196494 container init dd27e9fd63b78edc79aa18311d1ce2bb4e5a3d59899496c55dbd374360ecc9a2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6-deactivate, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, release=754, io.openshift.expose-services=, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, vcs-type=git, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., ceph=True, version=8) 2024-04-22T22:32:55.517 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:55 smithi086 podman[43867]: 2024-04-22 22:32:55.247612811 +0000 UTC m=+0.335603037 container start dd27e9fd63b78edc79aa18311d1ce2bb4e5a3d59899496c55dbd374360ecc9a2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6-deactivate, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, release=754, GIT_BRANCH=HEAD, architecture=x86_64, io.openshift.expose-services=, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-22T22:32:55.517 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:55 smithi086 podman[43867]: 2024-04-22 22:32:55.310187899 +0000 UTC m=+0.398178128 container attach dd27e9fd63b78edc79aa18311d1ce2bb4e5a3d59899496c55dbd374360ecc9a2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6-deactivate, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, GIT_CLEAN=True, ceph=True, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=) 2024-04-22T22:32:55.517 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:55 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:55.310+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:04.563374+0000 front 2024-04-22T22:32:04.563471+0000 (oldest deadline 2024-04-22T22:32:28.663454+0000) 2024-04-22T22:32:55.517 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:55 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:55.310+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:14.365427+0000 front 2024-04-22T22:32:14.365553+0000 (oldest deadline 2024-04-22T22:32:40.265418+0000) 2024-04-22T22:32:55.517 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:55 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:55.310+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6822 osd.2 since back 2024-04-22T22:32:20.266121+0000 front 2024-04-22T22:32:20.266059+0000 (oldest deadline 2024-04-22T22:32:44.365892+0000) 2024-04-22T22:32:55.517 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:55 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:55.310+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6830 osd.3 since back 2024-04-22T22:32:30.767548+0000 front 2024-04-22T22:32:30.767529+0000 (oldest deadline 2024-04-22T22:32:53.667419+0000) 2024-04-22T22:32:55.811 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:55 smithi086 podman[43867]: 2024-04-22 22:32:55.52848518 +0000 UTC m=+0.616475417 container died dd27e9fd63b78edc79aa18311d1ce2bb4e5a3d59899496c55dbd374360ecc9a2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, io.openshift.expose-services=, version=8, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870) 2024-04-22T22:32:56.051 DEBUG:teuthology.orchestra.run.smithi086:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@osd.6.service' 2024-04-22T22:32:56.073 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:56 smithi086 podman[43892]: 2024-04-22 22:32:56.030445768 +0000 UTC m=+0.496243781 container remove dd27e9fd63b78edc79aa18311d1ce2bb4e5a3d59899496c55dbd374360ecc9a2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, version=8, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, architecture=x86_64, vendor=Red Hat, Inc., release=754, maintainer=Guillaume Abrioux , RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-22T22:32:56.073 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:56 smithi086 systemd[1]: ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@osd.6.service: Deactivated successfully. 2024-04-22T22:32:56.073 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:56 smithi086 systemd[1]: Stopped Ceph osd.6 for 1ab1a3a8-00f6-11ef-bc93-c7b262605968. 2024-04-22T22:32:56.073 INFO:journalctl@ceph.osd.6.smithi086.stdout:Apr 22 22:32:56 smithi086 systemd[1]: ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@osd.6.service: Consumed 4.970s CPU time. 2024-04-22T22:32:56.086 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T22:32:56.087 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-04-22T22:32:56.087 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-04-22T22:32:56.087 DEBUG:teuthology.orchestra.run.smithi086:> sudo systemctl stop ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@osd.7 2024-04-22T22:32:56.516 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:56 smithi086 systemd[1]: Stopping Ceph osd.7 for 1ab1a3a8-00f6-11ef-bc93-c7b262605968... 2024-04-22T22:32:56.516 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:56 smithi086 bash[43944]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd.7" found: no such container 2024-04-22T22:32:56.517 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:56 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:56.263+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:04.563374+0000 front 2024-04-22T22:32:04.563471+0000 (oldest deadline 2024-04-22T22:32:28.663454+0000) 2024-04-22T22:32:56.517 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:56 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:56.263+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:14.365427+0000 front 2024-04-22T22:32:14.365553+0000 (oldest deadline 2024-04-22T22:32:40.265418+0000) 2024-04-22T22:32:56.517 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:56 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:56.263+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6822 osd.2 since back 2024-04-22T22:32:20.266121+0000 front 2024-04-22T22:32:20.266059+0000 (oldest deadline 2024-04-22T22:32:44.365892+0000) 2024-04-22T22:32:56.517 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:56 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:56.263+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6830 osd.3 since back 2024-04-22T22:32:30.767548+0000 front 2024-04-22T22:32:30.767529+0000 (oldest deadline 2024-04-22T22:32:53.667419+0000) 2024-04-22T22:32:56.518 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:56 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:56.307+0000 7f2c97836700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-22T22:32:56.518 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:56 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:56.307+0000 7f2c97836700 -1 osd.7 95 *** Got signal Terminated *** 2024-04-22T22:32:56.518 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:56 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:56.307+0000 7f2c97836700 -1 osd.7 95 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-22T22:32:57.515 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:57 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:57.253+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:04.563374+0000 front 2024-04-22T22:32:04.563471+0000 (oldest deadline 2024-04-22T22:32:28.663454+0000) 2024-04-22T22:32:57.516 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:57 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:57.253+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:14.365427+0000 front 2024-04-22T22:32:14.365553+0000 (oldest deadline 2024-04-22T22:32:40.265418+0000) 2024-04-22T22:32:57.516 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:57 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:57.253+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6822 osd.2 since back 2024-04-22T22:32:20.266121+0000 front 2024-04-22T22:32:20.266059+0000 (oldest deadline 2024-04-22T22:32:44.365892+0000) 2024-04-22T22:32:57.516 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:57 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:57.253+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6830 osd.3 since back 2024-04-22T22:32:30.767548+0000 front 2024-04-22T22:32:30.767529+0000 (oldest deadline 2024-04-22T22:32:53.667419+0000) 2024-04-22T22:32:58.765 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:58 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:58.293+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:04.563374+0000 front 2024-04-22T22:32:04.563471+0000 (oldest deadline 2024-04-22T22:32:28.663454+0000) 2024-04-22T22:32:58.766 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:58 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:58.293+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:14.365427+0000 front 2024-04-22T22:32:14.365553+0000 (oldest deadline 2024-04-22T22:32:40.265418+0000) 2024-04-22T22:32:58.766 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:58 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:58.293+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6822 osd.2 since back 2024-04-22T22:32:20.266121+0000 front 2024-04-22T22:32:20.266059+0000 (oldest deadline 2024-04-22T22:32:44.365892+0000) 2024-04-22T22:32:58.766 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:58 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:58.293+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6830 osd.3 since back 2024-04-22T22:32:30.767548+0000 front 2024-04-22T22:32:30.767529+0000 (oldest deadline 2024-04-22T22:32:53.667419+0000) 2024-04-22T22:32:59.765 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:59 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:59.334+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:04.563374+0000 front 2024-04-22T22:32:04.563471+0000 (oldest deadline 2024-04-22T22:32:28.663454+0000) 2024-04-22T22:32:59.766 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:59 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:59.334+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:14.365427+0000 front 2024-04-22T22:32:14.365553+0000 (oldest deadline 2024-04-22T22:32:40.265418+0000) 2024-04-22T22:32:59.766 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:59 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:59.334+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6822 osd.2 since back 2024-04-22T22:32:20.266121+0000 front 2024-04-22T22:32:20.266059+0000 (oldest deadline 2024-04-22T22:32:44.365892+0000) 2024-04-22T22:32:59.766 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:32:59 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:32:59.334+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6830 osd.3 since back 2024-04-22T22:32:30.767548+0000 front 2024-04-22T22:32:30.767529+0000 (oldest deadline 2024-04-22T22:32:53.667419+0000) 2024-04-22T22:33:00.765 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:33:00 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:33:00.379+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6806 osd.0 since back 2024-04-22T22:32:04.563374+0000 front 2024-04-22T22:32:04.563471+0000 (oldest deadline 2024-04-22T22:32:28.663454+0000) 2024-04-22T22:33:00.766 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:33:00 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:33:00.379+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6814 osd.1 since back 2024-04-22T22:32:14.365427+0000 front 2024-04-22T22:32:14.365553+0000 (oldest deadline 2024-04-22T22:32:40.265418+0000) 2024-04-22T22:33:00.766 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:33:00 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:33:00.379+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6822 osd.2 since back 2024-04-22T22:32:20.266121+0000 front 2024-04-22T22:32:20.266059+0000 (oldest deadline 2024-04-22T22:32:44.365892+0000) 2024-04-22T22:33:00.766 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:33:00 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:33:00.379+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.67:6830 osd.3 since back 2024-04-22T22:32:30.767548+0000 front 2024-04-22T22:32:30.767529+0000 (oldest deadline 2024-04-22T22:32:53.667419+0000) 2024-04-22T22:33:00.766 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:33:00 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7[34732]: 2024-04-22T22:33:00.379+0000 7f2c949d2700 -1 osd.7 95 heartbeat_check: no reply from 172.21.15.86:6804 osd.4 since back 2024-04-22T22:32:37.768167+0000 front 2024-04-22T22:32:37.768185+0000 (oldest deadline 2024-04-22T22:32:59.468205+0000) 2024-04-22T22:33:01.901 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:33:01 smithi086 podman[43957]: 2024-04-22 22:33:01.620554028 +0000 UTC m=+5.418451362 container died fa3c88c96bae26a9126e3b83a642cde61375ac688b4dae4e4253dcb4473729b0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, release=754, name=centos-stream, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , vcs-type=git, io.openshift.expose-services=, version=8, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, vendor=Red Hat, Inc., ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-04-22T22:33:02.209 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:33:01 smithi086 podman[43957]: 2024-04-22 22:33:01.985182429 +0000 UTC m=+5.783079774 container cleanup fa3c88c96bae26a9126e3b83a642cde61375ac688b4dae4e4253dcb4473729b0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, vendor=Red Hat, Inc., GIT_CLEAN=True, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, release=754) 2024-04-22T22:33:02.210 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:33:01 smithi086 bash[43957]: ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7 2024-04-22T22:33:02.210 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:33:02 smithi086 bash[43987]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd.7" found: no such container 2024-04-22T22:33:02.466 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:33:02 smithi086 podman[43974]: 2024-04-22 22:33:02.211162415 +0000 UTC m=+0.583986302 container remove fa3c88c96bae26a9126e3b83a642cde61375ac688b4dae4e4253dcb4473729b0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, release=754, io.buildah.version=1.19.8, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, ceph=True, version=8, distribution-scope=public) 2024-04-22T22:33:02.466 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:33:02 smithi086 podman[44052]: 2024-04-22 22:33:02.361946421 +0000 UTC m=+0.113716863 container create 2225fb855f307090dd5478491445587c0a1e1ef7e595eaade181126e23f17f4c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7-deactivate, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, version=8, architecture=x86_64, com.redhat.component=centos-stream-container, GIT_CLEAN=True, maintainer=Guillaume Abrioux , release=754, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8) 2024-04-22T22:33:02.467 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:33:02 smithi086 podman[44052]: 2024-04-22 22:33:02.267010486 +0000 UTC m=+0.018780883 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-22T22:33:02.766 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:33:02 smithi086 podman[44052]: 2024-04-22 22:33:02.572336466 +0000 UTC m=+0.324106863 container init 2225fb855f307090dd5478491445587c0a1e1ef7e595eaade181126e23f17f4c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7-deactivate, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , vcs-type=git, RELEASE=HEAD, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, name=centos-stream, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, release=754, io.openshift.expose-services=) 2024-04-22T22:33:02.766 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:33:02 smithi086 podman[44052]: 2024-04-22 22:33:02.575518843 +0000 UTC m=+0.327289248 container start 2225fb855f307090dd5478491445587c0a1e1ef7e595eaade181126e23f17f4c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7-deactivate, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, io.openshift.expose-services=, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-22T22:33:02.766 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:33:02 smithi086 podman[44052]: 2024-04-22 22:33:02.655515152 +0000 UTC m=+0.407285566 container attach 2225fb855f307090dd5478491445587c0a1e1ef7e595eaade181126e23f17f4c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7-deactivate, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., distribution-scope=public, io.k8s.display-name=CentOS Stream 8, version=8, io.buildah.version=1.19.8, release=754, RELEASE=HEAD, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, com.redhat.component=centos-stream-container, name=centos-stream) 2024-04-22T22:33:03.149 INFO:journalctl@ceph.osd.7.smithi086.stdout:Apr 22 22:33:02 smithi086 podman[44052]: 2024-04-22 22:33:02.863395956 +0000 UTC m=+0.615166365 container died 2225fb855f307090dd5478491445587c0a1e1ef7e595eaade181126e23f17f4c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-osd-7-deactivate, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, distribution-scope=public, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., io.buildah.version=1.19.8) 2024-04-22T22:33:03.431 DEBUG:teuthology.orchestra.run.smithi086:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@osd.7.service' 2024-04-22T22:33:03.468 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T22:33:03.468 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-04-22T22:33:03.468 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-04-22T22:33:03.469 DEBUG:teuthology.orchestra.run.smithi086:> sudo systemctl stop ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@prometheus.a 2024-04-22T22:33:03.693 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:33:03 smithi086 systemd[1]: Stopping Ceph prometheus.a for 1ab1a3a8-00f6-11ef-bc93-c7b262605968... 2024-04-22T22:33:03.693 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:33:03 smithi086 bash[44132]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-22T22:33:03.985 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:33:03 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[39513]: ts=2024-04-22T22:33:03.694Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-22T22:33:03.985 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:33:03 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[39513]: ts=2024-04-22T22:33:03.695Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-22T22:33:03.986 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:33:03 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[39513]: ts=2024-04-22T22:33:03.695Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-22T22:33:03.986 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:33:03 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[39513]: ts=2024-04-22T22:33:03.695Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-22T22:33:03.986 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:33:03 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[39513]: ts=2024-04-22T22:33:03.695Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-22T22:33:03.986 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:33:03 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[39513]: ts=2024-04-22T22:33:03.695Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-22T22:33:03.986 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:33:03 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[39513]: ts=2024-04-22T22:33:03.695Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-22T22:33:03.986 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:33:03 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[39513]: ts=2024-04-22T22:33:03.695Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-22T22:33:03.986 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:33:03 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[39513]: ts=2024-04-22T22:33:03.695Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-22T22:33:03.986 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:33:03 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[39513]: ts=2024-04-22T22:33:03.695Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-22T22:33:03.987 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:33:03 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[39513]: ts=2024-04-22T22:33:03.695Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-22T22:33:03.987 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:33:03 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a[39513]: ts=2024-04-22T22:33:03.695Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-22T22:33:03.987 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:33:03 smithi086 podman[44144]: 2024-04-22 22:33:03.704982013 +0000 UTC m=+0.125610539 container died 8531ba0d81dc2597a5694b7ee4b7258d37affd4db96bd904c4d07cdc9c949fb0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T22:33:04.266 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:33:04 smithi086 podman[44144]: 2024-04-22 22:33:04.086131841 +0000 UTC m=+0.506760363 container cleanup 8531ba0d81dc2597a5694b7ee4b7258d37affd4db96bd904c4d07cdc9c949fb0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-22T22:33:04.266 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:33:04 smithi086 bash[44144]: ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus-a 2024-04-22T22:33:04.266 INFO:journalctl@ceph.prometheus.a.smithi086.stdout:Apr 22 22:33:04 smithi086 bash[44170]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-22T22:33:04.307 DEBUG:teuthology.orchestra.run.smithi086:> sudo pkill -f 'journalctl -f -n 0 -u ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@prometheus.a.service' 2024-04-22T22:33:04.341 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-22T22:33:04.342 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-04-22T22:33:04.342 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 --force --keep-logs 2024-04-22T22:33:06.058 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:33:05 smithi067 systemd[1]: Stopping Ceph node-exporter.a for 1ab1a3a8-00f6-11ef-bc93-c7b262605968... 2024-04-22T22:33:06.058 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:33:05 smithi067 bash[54958]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter.a" found: no such container 2024-04-22T22:33:06.372 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:33:06 smithi067 podman[54968]: 2024-04-22 22:33:06.059216097 +0000 UTC m=+0.119697189 container died 865795666bfc7a806edaa0c6335ec7dd7a71c9b9509f8c3466a065404a8b5c7c (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T22:33:06.372 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:33:06 smithi067 podman[54968]: 2024-04-22 22:33:06.373695144 +0000 UTC m=+0.434176235 container cleanup 865795666bfc7a806edaa0c6335ec7dd7a71c9b9509f8c3466a065404a8b5c7c (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T22:33:06.767 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:33:06 smithi067 bash[54968]: ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a 2024-04-22T22:33:06.767 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:33:06 smithi067 podman[54980]: 2024-04-22 22:33:06.557897316 +0000 UTC m=+0.502615331 container remove 865795666bfc7a806edaa0c6335ec7dd7a71c9b9509f8c3466a065404a8b5c7c (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-22T22:33:06.767 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:33:06 smithi067 systemd[1]: ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-22T22:33:06.767 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:33:06 smithi067 bash[54996]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter.a" found: no such container 2024-04-22T22:33:06.767 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:33:06 smithi067 systemd[1]: ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-22T22:33:06.767 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:33:06 smithi067 systemd[1]: Stopped Ceph node-exporter.a for 1ab1a3a8-00f6-11ef-bc93-c7b262605968. 2024-04-22T22:33:06.768 INFO:journalctl@ceph.node-exporter.a.smithi067.stdout:Apr 22 22:33:06 smithi067 systemd[1]: ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@node-exporter.a.service: Consumed 4.384s CPU time. 2024-04-22T22:33:07.267 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:33:06 smithi067 systemd[1]: Stopping Ceph alertmanager.a for 1ab1a3a8-00f6-11ef-bc93-c7b262605968... 2024-04-22T22:33:07.267 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:33:06 smithi067 bash[55050]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager.a" found: no such container 2024-04-22T22:33:07.267 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:33:06 smithi067 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager-a[46701]: level=info ts=2024-04-22T22:33:06.910Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-04-22T22:33:07.267 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:33:06 smithi067 podman[55064]: 2024-04-22 22:33:06.932502713 +0000 UTC m=+0.117328537 container died 9d0e1f18c1cbfb68a5b722204cdfa86350fc0ef104a7cbf7b747d20e8973de8c (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T22:33:07.560 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:33:07 smithi067 podman[55064]: 2024-04-22 22:33:07.377307839 +0000 UTC m=+0.562133660 container cleanup 9d0e1f18c1cbfb68a5b722204cdfa86350fc0ef104a7cbf7b747d20e8973de8c (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T22:33:07.561 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:33:07 smithi067 bash[55064]: ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager-a 2024-04-22T22:33:07.939 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:33:07 smithi067 podman[55078]: 2024-04-22 22:33:07.561663272 +0000 UTC m=+0.626566934 container remove 9d0e1f18c1cbfb68a5b722204cdfa86350fc0ef104a7cbf7b747d20e8973de8c (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-22T22:33:07.939 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:33:07 smithi067 podman[55078]: 2024-04-22 22:33:07.645792127 +0000 UTC m=+0.710695790 volume remove 2dc21b77266e00edcde2c70c7b8a63a66aecda4229b642c5796e84d02a8715fa 2024-04-22T22:33:07.940 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:33:07 smithi067 bash[55090]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-alertmanager.a" found: no such container 2024-04-22T22:33:07.940 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:33:07 smithi067 systemd[1]: ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-22T22:33:07.940 INFO:journalctl@ceph.alertmanager.a.smithi067.stdout:Apr 22 22:33:07 smithi067 systemd[1]: Stopped Ceph alertmanager.a for 1ab1a3a8-00f6-11ef-bc93-c7b262605968. 2024-04-22T22:33:30.733 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 --force --keep-logs 2024-04-22T22:33:32.516 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:33:32 smithi086 systemd[1]: Stopping Ceph node-exporter.b for 1ab1a3a8-00f6-11ef-bc93-c7b262605968... 2024-04-22T22:33:32.516 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:33:32 smithi086 bash[44510]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter.b" found: no such container 2024-04-22T22:33:32.517 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:33:32 smithi086 podman[44523]: 2024-04-22 22:33:32.398229654 +0000 UTC m=+0.106089860 container died bc12658d1c9ca27b743aff7e4998c7c3f1a128c8a72903646e95eb170c0b1ba1 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T22:33:33.012 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:33:32 smithi086 podman[44523]: 2024-04-22 22:33:32.737925718 +0000 UTC m=+0.445785923 container cleanup bc12658d1c9ca27b743aff7e4998c7c3f1a128c8a72903646e95eb170c0b1ba1 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T22:33:33.012 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:33:32 smithi086 bash[44523]: ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b 2024-04-22T22:33:33.012 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:33:32 smithi086 bash[44550]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter.b" found: no such container 2024-04-22T22:33:33.266 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:33:33 smithi086 systemd[1]: Stopping Ceph grafana.a for 1ab1a3a8-00f6-11ef-bc93-c7b262605968... 2024-04-22T22:33:33.266 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:33:33 smithi086 bash[44603]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana.a" found: no such container 2024-04-22T22:33:33.266 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:33:33 smithi086 podman[44537]: 2024-04-22 22:33:33.01370606 +0000 UTC m=+0.619724386 container remove bc12658d1c9ca27b743aff7e4998c7c3f1a128c8a72903646e95eb170c0b1ba1 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-22T22:33:33.267 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:33:33 smithi086 systemd[1]: ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-22T22:33:33.267 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:33:33 smithi086 systemd[1]: ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-22T22:33:33.267 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:33:33 smithi086 systemd[1]: Stopped Ceph node-exporter.b for 1ab1a3a8-00f6-11ef-bc93-c7b262605968. 2024-04-22T22:33:33.267 INFO:journalctl@ceph.node-exporter.b.smithi086.stdout:Apr 22 22:33:33 smithi086 systemd[1]: ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@node-exporter.b.service: Consumed 4.169s CPU time. 2024-04-22T22:33:33.566 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:33:33 smithi086 ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a[38707]: t=2024-04-22T22:33:33+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-04-22T22:33:33.566 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:33:33 smithi086 podman[44616]: 2024-04-22 22:33:33.326892032 +0000 UTC m=+0.103368836 container died 5d6c83f8480c9a452d14aab2194c813ff3d4be802405ce2ea435a5d459b911b3 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a, io.openshift.tags=base rhel8, release=236.1648460182, architecture=x86_64, io.buildah.version=1.24.2, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, name=ubi8, build-date=2022-03-28T10:36:18.413762, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., version=8.5, description=Ceph Grafana Container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, maintainer=Paul Cuzner , summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, io.openshift.expose-services=) 2024-04-22T22:33:33.915 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:33:33 smithi086 podman[44616]: 2024-04-22 22:33:33.641223197 +0000 UTC m=+0.417700009 container cleanup 5d6c83f8480c9a452d14aab2194c813ff3d4be802405ce2ea435a5d459b911b3 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, name=ubi8, release=236.1648460182, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, io.buildah.version=1.24.2, architecture=x86_64, build-date=2022-03-28T10:36:18.413762, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, vendor=Red Hat, Inc., version=8.5, maintainer=Paul Cuzner ) 2024-04-22T22:33:33.916 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:33:33 smithi086 bash[44616]: ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a 2024-04-22T22:33:33.916 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:33:33 smithi086 bash[44642]: Error: no container with name or ID "ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana.a" found: no such container 2024-04-22T22:33:34.220 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:33:33 smithi086 podman[44630]: 2024-04-22 22:33:33.917078158 +0000 UTC m=+0.587259580 container remove 5d6c83f8480c9a452d14aab2194c813ff3d4be802405ce2ea435a5d459b911b3 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968-grafana-a, release=236.1648460182, io.openshift.expose-services=, version=8.5, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , distribution-scope=public, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., build-date=2022-03-28T10:36:18.413762, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, io.buildah.version=1.24.2, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-04-22T22:33:34.220 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:33:33 smithi086 systemd[1]: ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@grafana.a.service: Deactivated successfully. 2024-04-22T22:33:34.220 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:33:33 smithi086 systemd[1]: Stopped Ceph grafana.a for 1ab1a3a8-00f6-11ef-bc93-c7b262605968. 2024-04-22T22:33:34.220 INFO:journalctl@ceph.grafana.a.smithi086.stdout:Apr 22 22:33:33 smithi086 systemd[1]: ceph-1ab1a3a8-00f6-11ef-bc93-c7b262605968@grafana.a.service: Consumed 3.561s CPU time. 2024-04-22T22:33:45.305 DEBUG:teuthology.orchestra.run.smithi067:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-22T22:33:45.338 DEBUG:teuthology.orchestra.run.smithi086:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-22T22:33:45.364 INFO:tasks.cephadm:Archiving crash dumps... 2024-04-22T22:33:45.366 DEBUG:teuthology.misc:Transferring archived files from smithi067:/var/lib/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-04-22_21:08:03-orch-squid-distro-default-smithi/7668863/remote/smithi067/crash 2024-04-22T22:33:45.366 DEBUG:teuthology.orchestra.run.smithi067:> sudo tar c -f - -C /var/lib/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/crash -- . 2024-04-22T22:33:45.429 INFO:teuthology.orchestra.run.smithi067.stderr:tar: /var/lib/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/crash: Cannot open: No such file or directory 2024-04-22T22:33:45.430 INFO:teuthology.orchestra.run.smithi067.stderr:tar: Error is not recoverable: exiting now 2024-04-22T22:33:45.431 DEBUG:teuthology.misc:Transferring archived files from smithi086:/var/lib/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-04-22_21:08:03-orch-squid-distro-default-smithi/7668863/remote/smithi086/crash 2024-04-22T22:33:45.432 DEBUG:teuthology.orchestra.run.smithi086:> sudo tar c -f - -C /var/lib/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/crash -- . 2024-04-22T22:33:45.466 INFO:teuthology.orchestra.run.smithi086.stderr:tar: /var/lib/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/crash: Cannot open: No such file or directory 2024-04-22T22:33:45.466 INFO:teuthology.orchestra.run.smithi086.stderr:tar: Error is not recoverable: exiting now 2024-04-22T22:33:45.467 INFO:tasks.cephadm:Checking cluster log for badness... 2024-04-22T22:33:45.467 DEBUG:teuthology.orchestra.run.smithi067:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-22T22:33:45.497 INFO:tasks.cephadm:Compressing logs... 2024-04-22T22:33:45.498 DEBUG:teuthology.orchestra.run.smithi067:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-22T22:33:45.542 DEBUG:teuthology.orchestra.run.smithi086:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-22T22:33:45.564 INFO:teuthology.orchestra.run.smithi067.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-22T22:33:45.565 INFO:teuthology.orchestra.run.smithi067.stderr:: No such file or directory 2024-04-22T22:33:45.565 INFO:teuthology.orchestra.run.smithi067.stderr:gzip -5 --verbose -- /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph.log 2024-04-22T22:33:45.565 INFO:teuthology.orchestra.run.smithi067.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-osd.2.log 2024-04-22T22:33:45.565 INFO:teuthology.orchestra.run.smithi067.stderr:/var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph.audit.log 2024-04-22T22:33:45.565 INFO:teuthology.orchestra.run.smithi067.stderr:/var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-osd.3.log 2024-04-22T22:33:45.565 INFO:teuthology.orchestra.run.smithi067.stderr:gzip -5 --verbose -- /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-osd.1.log 2024-04-22T22:33:45.566 INFO:teuthology.orchestra.run.smithi067.stderr:/var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph.audit.log: /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-mgr.y.log 2024-04-22T22:33:45.566 INFO:teuthology.orchestra.run.smithi067.stderr:/var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-osd.0.log 2024-04-22T22:33:45.572 INFO:teuthology.orchestra.run.smithi067.stderr:/var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-mgr.y.log: 93.6% -- replaced with /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph.log.gz 2024-04-22T22:33:45.572 INFO:teuthology.orchestra.run.smithi067.stderr:gzip -5 --verbose -- /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-mon.c.log 2024-04-22T22:33:45.572 INFO:teuthology.orchestra.run.smithi067.stderr:/var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-osd.0.log: 93.9% -- replaced with /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph.audit.log.gz 2024-04-22T22:33:45.572 INFO:teuthology.orchestra.run.smithi067.stderr:gzip -5 --verbose -- /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-mon.a.log 2024-04-22T22:33:45.576 INFO:teuthology.orchestra.run.smithi086.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-22T22:33:45.576 INFO:teuthology.orchestra.run.smithi086.stderr:: No such file or directory 2024-04-22T22:33:45.576 INFO:teuthology.orchestra.run.smithi086.stderr:gzip -5 --verbose -- /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph.log 2024-04-22T22:33:45.576 INFO:teuthology.orchestra.run.smithi086.stderr:gzip -5 --verbose -- /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph.audit.log 2024-04-22T22:33:45.576 INFO:teuthology.orchestra.run.smithi086.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-osd.7.log 2024-04-22T22:33:45.577 INFO:teuthology.orchestra.run.smithi086.stderr:/var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph.log: /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-osd.5.log 2024-04-22T22:33:45.577 INFO:teuthology.orchestra.run.smithi086.stderr:/var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-osd.4.log 2024-04-22T22:33:45.577 INFO:teuthology.orchestra.run.smithi086.stderr:/var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-mgr.x.log 2024-04-22T22:33:45.577 INFO:teuthology.orchestra.run.smithi086.stderr:/var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- 88.7% -- replaced with /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph.log.gz 2024-04-22T22:33:45.577 INFO:teuthology.orchestra.run.smithi086.stderr: /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-osd.6.log 2024-04-22T22:33:45.577 INFO:teuthology.orchestra.run.smithi086.stderr:/var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-client.rgw.foo.smithi086.cofxku.log 2024-04-22T22:33:45.577 INFO:teuthology.orchestra.run.smithi086.stderr:/var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-osd.6.log: gzip 90.3% -- replaced with /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph.audit.log.gz 2024-04-22T22:33:45.577 INFO:teuthology.orchestra.run.smithi086.stderr: -5 --verbose -- /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph.cephadm.log 2024-04-22T22:33:45.578 INFO:teuthology.orchestra.run.smithi086.stderr:gzip -5 --verbose -- /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-volume.log 2024-04-22T22:33:45.578 INFO:teuthology.orchestra.run.smithi086.stderr:/var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-client.rgw.foo.smithi086.cofxku.log: /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph.cephadm.log: 62.4% -- replaced with /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-client.rgw.foo.smithi086.cofxku.log.gz 2024-04-22T22:33:45.578 INFO:teuthology.orchestra.run.smithi086.stderr: 80.8% -- replaced with /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph.cephadm.log.gzgzip -5 --verbose -- /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-mon.b.log 2024-04-22T22:33:45.578 INFO:teuthology.orchestra.run.smithi086.stderr: 2024-04-22T22:33:45.578 INFO:teuthology.orchestra.run.smithi086.stderr:/var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-volume.log: /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-mon.b.log: 90.6% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-22T22:33:45.578 INFO:teuthology.orchestra.run.smithi086.stderr: 91.4% -- replaced with /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-mgr.x.log.gz 2024-04-22T22:33:45.578 INFO:teuthology.orchestra.run.smithi086.stderr: 90.7% -- replaced with /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-volume.log.gz 2024-04-22T22:33:45.580 INFO:teuthology.orchestra.run.smithi067.stderr:/var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-mon.c.log: 91.5% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-22T22:33:45.580 INFO:teuthology.orchestra.run.smithi067.stderr:gzip -5 --verbose -- /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph.cephadm.log 2024-04-22T22:33:45.583 INFO:teuthology.orchestra.run.smithi067.stderr:/var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/tcmu-runner.log 2024-04-22T22:33:45.584 INFO:teuthology.orchestra.run.smithi067.stderr:/var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph.cephadm.log: 88.9% -- replaced with /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph.cephadm.log.gz 2024-04-22T22:33:45.596 INFO:teuthology.orchestra.run.smithi067.stderr:gzip -5 --verbose -- /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-volume.log 2024-04-22T22:33:45.597 INFO:teuthology.orchestra.run.smithi067.stderr:/var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/tcmu-runner.log: 66.6% -- replaced with /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/tcmu-runner.log.gz 2024-04-22T22:33:45.605 INFO:teuthology.orchestra.run.smithi067.stderr:gzip -5 --verbose -- /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-client.rgw.foo.smithi067.bekyzf.log 2024-04-22T22:33:45.619 INFO:teuthology.orchestra.run.smithi067.stderr:/var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-volume.log: /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-client.rgw.foo.smithi067.bekyzf.log: 62.4% -- replaced with /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-client.rgw.foo.smithi067.bekyzf.log.gz 2024-04-22T22:33:45.624 INFO:teuthology.orchestra.run.smithi067.stderr: 90.9% -- replaced with /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-volume.log.gz 2024-04-22T22:33:45.712 INFO:teuthology.orchestra.run.smithi086.stderr: 92.3% -- replaced with /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-mon.b.log.gz 2024-04-22T22:33:45.758 INFO:teuthology.orchestra.run.smithi067.stderr: 89.2% -- replaced with /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-mgr.y.log.gz 2024-04-22T22:33:45.779 INFO:teuthology.orchestra.run.smithi067.stderr: 92.3% -- replaced with /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-mon.c.log.gz 2024-04-22T22:33:45.968 INFO:teuthology.orchestra.run.smithi086.stderr: 93.0% -- replaced with /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-osd.6.log.gz 2024-04-22T22:33:45.978 INFO:teuthology.orchestra.run.smithi086.stderr: 93.7% -- replaced with /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-osd.7.log.gz 2024-04-22T22:33:45.984 INFO:teuthology.orchestra.run.smithi067.stderr: 93.1% -- replaced with /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-osd.2.log.gz 2024-04-22T22:33:45.985 INFO:teuthology.orchestra.run.smithi086.stderr: 93.2% -- replaced with /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-osd.5.log.gz 2024-04-22T22:33:46.082 INFO:teuthology.orchestra.run.smithi086.stderr: 93.4% -- replaced with /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-osd.4.log.gz 2024-04-22T22:33:46.084 INFO:teuthology.orchestra.run.smithi086.stderr: 2024-04-22T22:33:46.084 INFO:teuthology.orchestra.run.smithi086.stderr:real 0m0.527s 2024-04-22T22:33:46.084 INFO:teuthology.orchestra.run.smithi086.stderr:user 0m1.842s 2024-04-22T22:33:46.084 INFO:teuthology.orchestra.run.smithi086.stderr:sys 0m0.094s 2024-04-22T22:33:46.109 INFO:teuthology.orchestra.run.smithi067.stderr: 90.9% -- replaced with /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-mon.a.log.gz 2024-04-22T22:33:46.118 INFO:teuthology.orchestra.run.smithi067.stderr: 93.3% -- replaced with /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-osd.0.log.gz 2024-04-22T22:33:46.165 INFO:teuthology.orchestra.run.smithi067.stderr: 93.4% -- replaced with /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-osd.3.log.gz 2024-04-22T22:33:46.207 INFO:teuthology.orchestra.run.smithi067.stderr: 93.3% -- replaced with /var/log/ceph/1ab1a3a8-00f6-11ef-bc93-c7b262605968/ceph-osd.1.log.gz 2024-04-22T22:33:46.209 INFO:teuthology.orchestra.run.smithi067.stderr: 2024-04-22T22:33:46.209 INFO:teuthology.orchestra.run.smithi067.stderr:real 0m0.657s 2024-04-22T22:33:46.209 INFO:teuthology.orchestra.run.smithi067.stderr:user 0m3.079s 2024-04-22T22:33:46.209 INFO:teuthology.orchestra.run.smithi067.stderr:sys 0m0.112s 2024-04-22T22:33:46.210 INFO:tasks.cephadm:Archiving logs... 2024-04-22T22:33:46.210 DEBUG:teuthology.misc:Transferring archived files from smithi067:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-22_21:08:03-orch-squid-distro-default-smithi/7668863/remote/smithi067/log 2024-04-22T22:33:46.211 DEBUG:teuthology.orchestra.run.smithi067:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-22T22:33:46.663 DEBUG:teuthology.misc:Transferring archived files from smithi086:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-22_21:08:03-orch-squid-distro-default-smithi/7668863/remote/smithi086/log 2024-04-22T22:33:46.664 DEBUG:teuthology.orchestra.run.smithi086:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-22T22:33:46.979 INFO:tasks.cephadm:Removing cluster... 2024-04-22T22:33:46.979 DEBUG:teuthology.orchestra.run.smithi067:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 --force 2024-04-22T22:33:47.263 DEBUG:teuthology.orchestra.run.smithi086:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 --force 2024-04-22T22:33:47.549 INFO:tasks.cephadm:Removing cephadm ... 2024-04-22T22:33:47.549 DEBUG:teuthology.orchestra.run.smithi067:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-22T22:33:47.565 DEBUG:teuthology.orchestra.run.smithi086:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-22T22:33:47.580 INFO:tasks.cephadm:Teardown complete 2024-04-22T22:33:47.580 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-04-22T22:33:47.591 INFO:teuthology.task.clock:Checking final clock skew... 2024-04-22T22:33:47.592 DEBUG:teuthology.orchestra.run.smithi067:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-22T22:33:47.608 DEBUG:teuthology.orchestra.run.smithi086:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-22T22:33:47.620 INFO:teuthology.orchestra.run.smithi067.stderr:bash: line 1: ntpq: command not found 2024-04-22T22:33:47.625 INFO:teuthology.orchestra.run.smithi067.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-22T22:33:47.625 INFO:teuthology.orchestra.run.smithi067.stdout:=============================================================================== 2024-04-22T22:33:47.625 INFO:teuthology.orchestra.run.smithi067.stdout:^- hv01.front.sepia.ceph.com 2 6 377 1 +697us[ +697us] +/- 574ms 2024-04-22T22:33:47.625 INFO:teuthology.orchestra.run.smithi067.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 3 -96us[ -96us] +/- 51ms 2024-04-22T22:33:47.626 INFO:teuthology.orchestra.run.smithi067.stdout:^* hv03.front.sepia.ceph.com 3 6 377 4 +127us[ +106us] +/- 62ms 2024-04-22T22:33:47.626 INFO:teuthology.orchestra.run.smithi067.stdout:^? hv04.front.sepia.ceph.com 0 8 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T22:33:47.633 INFO:teuthology.orchestra.run.smithi086.stderr:bash: line 1: ntpq: command not found 2024-04-22T22:33:47.638 INFO:teuthology.orchestra.run.smithi086.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-22T22:33:47.638 INFO:teuthology.orchestra.run.smithi086.stdout:=============================================================================== 2024-04-22T22:33:47.638 INFO:teuthology.orchestra.run.smithi086.stdout:^- hv01.front.sepia.ceph.com 2 6 377 1 +700us[ +700us] +/- 574ms 2024-04-22T22:33:47.638 INFO:teuthology.orchestra.run.smithi086.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 2 -91us[ -91us] +/- 51ms 2024-04-22T22:33:47.638 INFO:teuthology.orchestra.run.smithi086.stdout:^* hv03.front.sepia.ceph.com 3 6 377 3 +132us[ +119us] +/- 62ms 2024-04-22T22:33:47.638 INFO:teuthology.orchestra.run.smithi086.stdout:^? hv04.front.sepia.ceph.com 0 8 0 - +0ns[ +0ns] +/- 0ns 2024-04-22T22:33:47.639 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-22T22:33:47.647 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-22T22:33:47.648 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-22T22:33:47.657 DEBUG:teuthology.orchestra.run.smithi067:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-22T22:33:47.737 DEBUG:teuthology.orchestra.run.smithi086:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-22T22:33:47.800 DEBUG:teuthology.orchestra.run.smithi067:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-22T22:33:47.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T22:33:47.827 DEBUG:teuthology.orchestra.run.smithi086:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-22T22:33:47.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T22:33:47.854 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-22T22:33:47.863 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-22T22:33:47.871 INFO:teuthology.task.internal:Duration was 1202.836202 seconds 2024-04-22T22:33:47.872 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-22T22:33:47.881 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-22T22:33:47.882 DEBUG:teuthology.orchestra.run.smithi067:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-22T22:33:47.884 DEBUG:teuthology.orchestra.run.smithi086:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-22T22:33:47.920 INFO:teuthology.orchestra.run.smithi067.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-22T22:33:47.929 INFO:teuthology.orchestra.run.smithi086.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-22T22:33:48.299 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-22T22:33:48.300 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi067.front.sepia.ceph.com 2024-04-22T22:33:48.300 DEBUG:teuthology.orchestra.run.smithi067:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-22T22:33:48.320 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi086.front.sepia.ceph.com 2024-04-22T22:33:48.321 DEBUG:teuthology.orchestra.run.smithi086:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-22T22:33:48.366 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-22T22:33:48.367 DEBUG:teuthology.orchestra.run.smithi067:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-22T22:33:48.369 DEBUG:teuthology.orchestra.run.smithi086:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-22T22:33:48.432 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-22T22:33:48.432 DEBUG:teuthology.orchestra.run.smithi067:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-22T22:33:48.435 DEBUG:teuthology.orchestra.run.smithi086:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-22T22:33:48.737 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-22T22:33:48.754 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-22T22:33:48.754 DEBUG:teuthology.orchestra.run.smithi067:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-22T22:33:48.779 DEBUG:teuthology.orchestra.run.smithi086:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-22T22:33:48.803 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-22T22:33:48.812 DEBUG:teuthology.orchestra.run.smithi067:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-22T22:33:48.822 DEBUG:teuthology.orchestra.run.smithi086:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-22T22:33:48.842 INFO:teuthology.orchestra.run.smithi067.stdout:kernel.core_pattern = core 2024-04-22T22:33:48.864 INFO:teuthology.orchestra.run.smithi086.stdout:kernel.core_pattern = core 2024-04-22T22:33:48.888 DEBUG:teuthology.orchestra.run.smithi067:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-22T22:33:48.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T22:33:48.919 DEBUG:teuthology.orchestra.run.smithi086:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-22T22:33:48.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-22T22:33:48.943 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-22T22:33:48.954 INFO:teuthology.task.internal:Transferring archived files... 2024-04-22T22:33:48.955 DEBUG:teuthology.misc:Transferring archived files from smithi067:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-22_21:08:03-orch-squid-distro-default-smithi/7668863/remote/smithi067 2024-04-22T22:33:48.955 DEBUG:teuthology.orchestra.run.smithi067:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-22T22:33:49.001 DEBUG:teuthology.misc:Transferring archived files from smithi086:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-22_21:08:03-orch-squid-distro-default-smithi/7668863/remote/smithi086 2024-04-22T22:33:49.002 DEBUG:teuthology.orchestra.run.smithi086:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-22T22:33:49.036 INFO:teuthology.task.internal:Removing archive directory... 2024-04-22T22:33:49.037 DEBUG:teuthology.orchestra.run.smithi067:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-22T22:33:49.040 DEBUG:teuthology.orchestra.run.smithi086:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-22T22:33:49.085 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-22T22:33:49.097 INFO:teuthology.task.internal:Not uploading archives. 2024-04-22T22:33:49.097 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-22T22:33:49.107 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-22T22:33:49.107 DEBUG:teuthology.orchestra.run.smithi067:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-22T22:33:49.110 DEBUG:teuthology.orchestra.run.smithi086:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-22T22:33:49.122 INFO:teuthology.orchestra.run.smithi067.stdout: 262213 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 22 22:33 /home/ubuntu/cephtest 2024-04-22T22:33:49.139 INFO:teuthology.orchestra.run.smithi086.stdout: 262213 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 22 22:33 /home/ubuntu/cephtest 2024-04-22T22:33:49.140 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-22T22:33:49.150 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-22T22:33:49.169 INFO:teuthology.nuke:Checking targets against current locks 2024-04-22T22:33:49.197 DEBUG:teuthology.nuke:shortname: smithi067 2024-04-22T22:33:49.198 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-22T22:33:49.223 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi067.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-22_21:08:03-orch-squid-distro-default-smithi/7668863', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-22 22:05:57.209905', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKtUaSojXmXB8EKXCcQdK+MdcQRTSMFEMXHra93PmKHxhrNISkr9m2nCZkftJhuIK2PX26IMVeIRLb1OWg1BSZE='} 2024-04-22T22:33:49.227 DEBUG:teuthology.nuke:shortname: smithi086 2024-04-22T22:33:49.228 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-22T22:33:49.250 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi086.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-22_21:08:03-orch-squid-distro-default-smithi/7668863', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-22 22:05:57.211382', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKzmqenF1jgMllBMhNcGRN1q0AhIAk99xkSim47SIa60lF4VTPqScAaVnw18Sr/mbRC7Ald58Ha35xjeWVHAlxM='} 2024-04-22T22:33:49.282 INFO:teuthology.orchestra.console.smithi067:Power off 2024-04-22T22:33:49.282 DEBUG:teuthology.orchestra.console.smithi067:pexpect command: ipmitool -H smithi067.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-22T22:33:49.307 DEBUG:teuthology.orchestra.console.smithi067:power off output: Chassis Power Control: Down/Off 2024-04-22T22:33:49.308 DEBUG:teuthology.orchestra.console.smithi067:pexpect command: ipmitool -H smithi067.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-22T22:33:49.330 INFO:teuthology.orchestra.console.smithi086:Power off 2024-04-22T22:33:49.330 DEBUG:teuthology.orchestra.console.smithi086:pexpect command: ipmitool -H smithi086.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-22T22:33:49.342 DEBUG:teuthology.orchestra.console.smithi067:check power output: Chassis Power is on 2024-04-22T22:33:49.357 DEBUG:teuthology.orchestra.console.smithi086:power off output: Chassis Power Control: Down/Off 2024-04-22T22:33:49.357 DEBUG:teuthology.orchestra.console.smithi086:pexpect command: ipmitool -H smithi086.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-22T22:33:49.384 DEBUG:teuthology.orchestra.console.smithi086:check power output: Chassis Power is on 2024-04-22T22:33:53.343 DEBUG:teuthology.orchestra.console.smithi067:pexpect command: ipmitool -H smithi067.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-22T22:33:53.386 DEBUG:teuthology.orchestra.console.smithi086:pexpect command: ipmitool -H smithi086.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-22T22:33:53.456 DEBUG:teuthology.orchestra.console.smithi067:check power output: Chassis Power is on 2024-04-22T22:33:53.498 DEBUG:teuthology.orchestra.console.smithi086:check power output: Chassis Power is on 2024-04-22T22:33:57.458 DEBUG:teuthology.orchestra.console.smithi067:pexpect command: ipmitool -H smithi067.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-22T22:33:57.499 DEBUG:teuthology.orchestra.console.smithi086:pexpect command: ipmitool -H smithi086.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-22T22:33:57.569 DEBUG:teuthology.orchestra.console.smithi067:check power output: Chassis Power is off 2024-04-22T22:33:57.609 DEBUG:teuthology.orchestra.console.smithi086:check power output: Chassis Power is off 2024-04-22T22:33:57.671 INFO:teuthology.orchestra.console.smithi067:Power off completed 2024-04-22T22:33:57.710 INFO:teuthology.orchestra.console.smithi086:Power off completed 2024-04-22T22:33:57.816 INFO:teuthology.run:Summary data: description: orch/cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/repo_digest 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} duration: 1202.8362019062042 failure_reason: 'Command failed on smithi067 with status 1: ''sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 1ab1a3a8-00f6-11ef-bc93-c7b262605968 -e sha1=430e09df97c8fc7dc2b2ae424f68ed11366c540f -- bash -c \''ceph versions | jq -e \''"\''"\''.mgr | length == 2\''"\''"\''\''''' owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=082249e5b7704c90b5a2c8ede1d47f94 status: fail success: false 2024-04-22T22:33:57.816 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-22T22:33:57.918 INFO:teuthology.run:FAIL