2024-04-02T15:16:31.333 INFO:root:teuthology version: 0.0.1.dev262+ge691533 2024-04-02T15:16:31.333 DEBUG:teuthology.run:Teuthology command: teuthology --owner scheduled_gabrioux@teuthology --description orch:cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/on mon_election/classic} --archive /home/teuthworker/archive/gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi/7636172 --name gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi --verbose -- /home/teuthworker/archive/gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi/7636172/orig.config.yaml 2024-04-02T15:16:31.386 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-02T15:16:31.483 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi/7636172 branch: wip-guits-testing-2024-04-02-0703 description: orch:cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/on mon_election/classic} email: gabrioux@ibm.com first_in_suite: false job_id: '7636172' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi name: gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi no_nested_subset: false nuke-on-error: true os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-guits-testing-2024-04-02-0703 ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mgr/cephadm/use_agent: true mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: 3f3f3ed992429b85be729743ad3b39d52dd90744 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 3f3f3ed992429b85be729743ad3b39d52dd90744 workunit: branch: wip-guits-testing-2024-04-02-0703 sha1: 3f3f3ed992429b85be729743ad3b39d52dd90744 owner: scheduled_gabrioux@teuthology priority: 70 repo: https://git.ceph.com/ceph-ci.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b seed: 5637 sha1: 3f3f3ed992429b85be729743ad3b39d52dd90744 sleep_before_teardown: 0 subset: 1/8 suite: orch:cephadm suite_branch: wip-guits-testing-2024-04-02-0703 suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_3f3f3ed992429b85be729743ad3b39d52dd90744/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: 3f3f3ed992429b85be729743ad3b39d52dd90744 targets: smithi087.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGxSrXeQXaQfffNLByQxohUKCY/iDJzmwOw0ad1+nmK0inhxPAWMyoRY28TeUDt7GkYdTi34J5VcVGYzc+/gPxc= smithi106.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPa6mpToccO2yFYP2iF9WIzT2AlXu5XNyUVL5qlsL30EiS/EKea53h9/0CSO34PPwfLbbXDhVgYjx62pVH+V5ag= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 - cephadm.shell: env: - sha1 mon.a: - radosgw-admin realm create --rgw-realm=r --default - radosgw-admin zonegroup create --rgw-zonegroup=default --master --default - radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default - radosgw-admin period update --rgw-realm=r --commit - ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000 - ceph osd pool create foo - rbd pool init foo - ceph orch apply iscsi foo u p - sleep 180 - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph orch ps - ceph versions - ceph -s - ceph orch ls - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 2' - ceph mgr fail - sleep 180 - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph health detail - ceph -s - ceph mgr fail - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 1' - ceph mgr fail - sleep 180 - ceph orch ps - ceph orch ls - ceph versions - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph versions | jq -e '.mgr | length == 1' - ceph versions | jq -e '.mgr | keys' | grep $sha1 - ceph versions | jq -e '.overall | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 1' - ceph versions | jq -e '.mon | keys' | grep $sha1 - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 5' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 7' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 8' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 1' - ceph versions | jq -e '.osd | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.rgw | length == 1' - ceph versions | jq -e '.rgw | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 mon.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph versions - ceph orch upgrade status - ceph health detail - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - ceph orch ls | grep '^osd ' - cephadm.shell: mon.a: - ceph orch upgrade ls - ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0 - ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2 teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: e691533f9cbb33d85b2187bba20d7102f098636d timestamp: 2024-04-02_14:30:22 tube: smithi user: gabrioux verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-04-02T15:16:31.484 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_3f3f3ed992429b85be729743ad3b39d52dd90744/qa; will attempt to use it 2024-04-02T15:16:31.484 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_3f3f3ed992429b85be729743ad3b39d52dd90744/qa/tasks 2024-04-02T15:16:31.485 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-02T15:16:31.486 INFO:teuthology.task.internal:Checking packages... 2024-04-02T15:16:31.524 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '3f3f3ed992429b85be729743ad3b39d52dd90744' 2024-04-02T15:16:31.524 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-02T15:16:31.525 INFO:teuthology.packaging:ref: None 2024-04-02T15:16:31.525 INFO:teuthology.packaging:tag: None 2024-04-02T15:16:31.525 INFO:teuthology.packaging:branch: wip-guits-testing-2024-04-02-0703 2024-04-02T15:16:31.525 INFO:teuthology.packaging:sha1: 3f3f3ed992429b85be729743ad3b39d52dd90744 2024-04-02T15:16:31.525 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-guits-testing-2024-04-02-0703 2024-04-02T15:16:31.667 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-2592.g3f3f3ed9 2024-04-02T15:16:31.668 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-02T15:16:31.676 INFO:teuthology.task.internal:no buildpackages task found 2024-04-02T15:16:31.676 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-02T15:16:31.680 INFO:teuthology.task.internal:Saving configuration 2024-04-02T15:16:31.699 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-02T15:16:31.704 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-02T15:16:31.727 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi087.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi/7636172', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-02 15:11:28.953202', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGxSrXeQXaQfffNLByQxohUKCY/iDJzmwOw0ad1+nmK0inhxPAWMyoRY28TeUDt7GkYdTi34J5VcVGYzc+/gPxc='} 2024-04-02T15:16:31.752 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi106.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi/7636172', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-02 15:11:28.954825', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPa6mpToccO2yFYP2iF9WIzT2AlXu5XNyUVL5qlsL30EiS/EKea53h9/0CSO34PPwfLbbXDhVgYjx62pVH+V5ag='} 2024-04-02T15:16:31.752 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-02T15:16:31.758 INFO:teuthology.task.internal:roles: ubuntu@smithi087.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-04-02T15:16:31.758 INFO:teuthology.task.internal:roles: ubuntu@smithi106.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-04-02T15:16:31.759 INFO:teuthology.run_tasks:Running task console_log... 2024-04-02T15:16:31.840 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f283d8050d0>, signals=[15]) 2024-04-02T15:16:31.840 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-02T15:16:31.846 INFO:teuthology.task.internal:Opening connections... 2024-04-02T15:16:31.847 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi087.front.sepia.ceph.com 2024-04-02T15:16:31.848 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi087.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T15:16:31.988 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi106.front.sepia.ceph.com 2024-04-02T15:16:31.989 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi106.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T15:16:32.088 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-02T15:16:32.094 DEBUG:teuthology.orchestra.run.smithi087:> uname -m 2024-04-02T15:16:32.113 INFO:teuthology.orchestra.run.smithi087.stdout:x86_64 2024-04-02T15:16:32.113 DEBUG:teuthology.orchestra.run.smithi087:> cat /etc/os-release 2024-04-02T15:16:32.171 INFO:teuthology.orchestra.run.smithi087.stdout:NAME="CentOS Stream" 2024-04-02T15:16:32.171 INFO:teuthology.orchestra.run.smithi087.stdout:VERSION="9" 2024-04-02T15:16:32.171 INFO:teuthology.orchestra.run.smithi087.stdout:ID="centos" 2024-04-02T15:16:32.171 INFO:teuthology.orchestra.run.smithi087.stdout:ID_LIKE="rhel fedora" 2024-04-02T15:16:32.171 INFO:teuthology.orchestra.run.smithi087.stdout:VERSION_ID="9" 2024-04-02T15:16:32.171 INFO:teuthology.orchestra.run.smithi087.stdout:PLATFORM_ID="platform:el9" 2024-04-02T15:16:32.171 INFO:teuthology.orchestra.run.smithi087.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-02T15:16:32.171 INFO:teuthology.orchestra.run.smithi087.stdout:ANSI_COLOR="0;31" 2024-04-02T15:16:32.171 INFO:teuthology.orchestra.run.smithi087.stdout:LOGO="fedora-logo-icon" 2024-04-02T15:16:32.172 INFO:teuthology.orchestra.run.smithi087.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-02T15:16:32.172 INFO:teuthology.orchestra.run.smithi087.stdout:HOME_URL="https://centos.org/" 2024-04-02T15:16:32.172 INFO:teuthology.orchestra.run.smithi087.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-02T15:16:32.172 INFO:teuthology.orchestra.run.smithi087.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-02T15:16:32.172 INFO:teuthology.orchestra.run.smithi087.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-02T15:16:32.173 INFO:teuthology.lock.ops:Updating smithi087.front.sepia.ceph.com on lock server 2024-04-02T15:16:32.193 DEBUG:teuthology.orchestra.run.smithi106:> uname -m 2024-04-02T15:16:32.209 INFO:teuthology.orchestra.run.smithi106.stdout:x86_64 2024-04-02T15:16:32.210 DEBUG:teuthology.orchestra.run.smithi106:> cat /etc/os-release 2024-04-02T15:16:32.266 INFO:teuthology.orchestra.run.smithi106.stdout:NAME="CentOS Stream" 2024-04-02T15:16:32.266 INFO:teuthology.orchestra.run.smithi106.stdout:VERSION="9" 2024-04-02T15:16:32.266 INFO:teuthology.orchestra.run.smithi106.stdout:ID="centos" 2024-04-02T15:16:32.266 INFO:teuthology.orchestra.run.smithi106.stdout:ID_LIKE="rhel fedora" 2024-04-02T15:16:32.266 INFO:teuthology.orchestra.run.smithi106.stdout:VERSION_ID="9" 2024-04-02T15:16:32.266 INFO:teuthology.orchestra.run.smithi106.stdout:PLATFORM_ID="platform:el9" 2024-04-02T15:16:32.266 INFO:teuthology.orchestra.run.smithi106.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-02T15:16:32.266 INFO:teuthology.orchestra.run.smithi106.stdout:ANSI_COLOR="0;31" 2024-04-02T15:16:32.266 INFO:teuthology.orchestra.run.smithi106.stdout:LOGO="fedora-logo-icon" 2024-04-02T15:16:32.267 INFO:teuthology.orchestra.run.smithi106.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-02T15:16:32.267 INFO:teuthology.orchestra.run.smithi106.stdout:HOME_URL="https://centos.org/" 2024-04-02T15:16:32.267 INFO:teuthology.orchestra.run.smithi106.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-02T15:16:32.267 INFO:teuthology.orchestra.run.smithi106.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-02T15:16:32.267 INFO:teuthology.orchestra.run.smithi106.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-02T15:16:32.267 INFO:teuthology.lock.ops:Updating smithi106.front.sepia.ceph.com on lock server 2024-04-02T15:16:32.287 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-02T15:16:32.296 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-02T15:16:32.313 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-02T15:16:32.314 DEBUG:teuthology.orchestra.run.smithi087:> test '!' -e /home/ubuntu/cephtest 2024-04-02T15:16:32.316 DEBUG:teuthology.orchestra.run.smithi106:> test '!' -e /home/ubuntu/cephtest 2024-04-02T15:16:32.333 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-02T15:16:32.339 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-02T15:16:32.339 DEBUG:teuthology.orchestra.run.smithi087:> test -z $(ls -A /var/lib/ceph) 2024-04-02T15:16:32.374 DEBUG:teuthology.orchestra.run.smithi106:> test -z $(ls -A /var/lib/ceph) 2024-04-02T15:16:32.402 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-02T15:16:32.454 INFO:teuthology.run_tasks:Running task kernel... 2024-04-02T15:16:32.468 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2024-04-02T15:16:32.468 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2024-04-02T15:16:32.469 DEBUG:teuthology.orchestra.run.smithi087:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-02T15:16:32.469 DEBUG:teuthology.orchestra.run.smithi106:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-02T15:16:32.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-02T15:16:32.485 DEBUG:teuthology.orchestra.run.smithi106:> uname -r 2024-04-02T15:16:32.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-02T15:16:32.486 DEBUG:teuthology.orchestra.run.smithi087:> uname -r 2024-04-02T15:16:32.542 INFO:teuthology.orchestra.run.smithi106.stdout:5.14.0-361.el9.x86_64 2024-04-02T15:16:32.543 INFO:teuthology.task.kernel:Running kernel on smithi106: 5.14.0-361.el9.x86_64 2024-04-02T15:16:32.543 DEBUG:teuthology.orchestra.run.smithi106:> sudo yum install -y kernel 2024-04-02T15:16:32.544 INFO:teuthology.orchestra.run.smithi087.stdout:5.14.0-361.el9.x86_64 2024-04-02T15:16:32.544 INFO:teuthology.task.kernel:Running kernel on smithi087: 5.14.0-361.el9.x86_64 2024-04-02T15:16:32.544 DEBUG:teuthology.orchestra.run.smithi087:> sudo yum install -y kernel 2024-04-02T15:16:34.630 INFO:teuthology.orchestra.run.smithi106.stdout:CentOS-9 - AppStream 14 MB/s | 6.7 MB 00:00 2024-04-02T15:16:34.649 INFO:teuthology.orchestra.run.smithi087.stdout:CentOS-9 - AppStream 15 MB/s | 6.7 MB 00:00 2024-04-02T15:16:36.292 INFO:teuthology.orchestra.run.smithi106.stdout:CentOS-9 - BaseOS 3.3 MB/s | 2.0 MB 00:00 2024-04-02T15:16:36.350 INFO:teuthology.orchestra.run.smithi087.stdout:CentOS-9 - BaseOS 3.4 MB/s | 2.0 MB 00:00 2024-04-02T15:16:37.458 INFO:teuthology.orchestra.run.smithi087.stdout:CentOS Stream 9 - BaseOS 10 MB/s | 8.0 MB 00:00 2024-04-02T15:16:38.704 INFO:teuthology.orchestra.run.smithi106.stdout:CentOS Stream 9 - BaseOS 3.8 MB/s | 8.0 MB 00:02 2024-04-02T15:16:40.308 INFO:teuthology.orchestra.run.smithi087.stdout:CentOS Stream 9 - AppStream 11 MB/s | 19 MB 00:01 2024-04-02T15:16:41.432 INFO:teuthology.orchestra.run.smithi106.stdout:CentOS Stream 9 - AppStream 11 MB/s | 19 MB 00:01 2024-04-02T15:16:46.334 INFO:teuthology.orchestra.run.smithi087.stdout:CentOS Stream 9 - CRB 2.4 MB/s | 6.1 MB 00:02 2024-04-02T15:16:48.102 INFO:teuthology.orchestra.run.smithi087.stdout:CentOS Stream 9 - Extras packages 25 kB/s | 16 kB 00:00 2024-04-02T15:16:48.331 INFO:teuthology.orchestra.run.smithi106.stdout:CentOS Stream 9 - CRB 1.8 MB/s | 6.1 MB 00:03 2024-04-02T15:16:49.063 INFO:teuthology.orchestra.run.smithi087.stdout:Extra Packages for Enterprise Linux 25 MB/s | 21 MB 00:00 2024-04-02T15:16:50.076 INFO:teuthology.orchestra.run.smithi106.stdout:CentOS Stream 9 - Extras packages 26 kB/s | 16 kB 00:00 2024-04-02T15:16:51.537 INFO:teuthology.orchestra.run.smithi106.stdout:Extra Packages for Enterprise Linux 16 MB/s | 21 MB 00:01 2024-04-02T15:16:54.037 INFO:teuthology.orchestra.run.smithi087.stdout:lab-extras 12 kB/s | 1.7 kB 00:00 2024-04-02T15:16:56.293 INFO:teuthology.orchestra.run.smithi087.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-02T15:16:56.293 INFO:teuthology.orchestra.run.smithi087.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-04-02T15:16:56.410 INFO:teuthology.orchestra.run.smithi087.stdout:Dependencies resolved. 2024-04-02T15:16:56.422 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-02T15:16:56.423 INFO:teuthology.orchestra.run.smithi087.stdout: Package Arch Version Repository Size 2024-04-02T15:16:56.423 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-02T15:16:56.423 INFO:teuthology.orchestra.run.smithi087.stdout:Installing: 2024-04-02T15:16:56.423 INFO:teuthology.orchestra.run.smithi087.stdout: kernel x86_64 5.14.0-434.el9 CentOS-BaseOS 192 k 2024-04-02T15:16:56.423 INFO:teuthology.orchestra.run.smithi087.stdout:Installing dependencies: 2024-04-02T15:16:56.423 INFO:teuthology.orchestra.run.smithi087.stdout: kernel-core x86_64 5.14.0-434.el9 CentOS-BaseOS 15 M 2024-04-02T15:16:56.423 INFO:teuthology.orchestra.run.smithi087.stdout: kernel-modules x86_64 5.14.0-434.el9 CentOS-BaseOS 34 M 2024-04-02T15:16:56.423 INFO:teuthology.orchestra.run.smithi087.stdout: kernel-modules-core x86_64 5.14.0-434.el9 CentOS-BaseOS 28 M 2024-04-02T15:16:56.423 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-02T15:16:56.423 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction Summary 2024-04-02T15:16:56.423 INFO:teuthology.orchestra.run.smithi087.stdout:================================================================================ 2024-04-02T15:16:56.423 INFO:teuthology.orchestra.run.smithi087.stdout:Install 4 Packages 2024-04-02T15:16:56.424 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-02T15:16:56.424 INFO:teuthology.orchestra.run.smithi087.stdout:Total download size: 76 M 2024-04-02T15:16:56.424 INFO:teuthology.orchestra.run.smithi087.stdout:Installed size: 123 M 2024-04-02T15:16:56.424 INFO:teuthology.orchestra.run.smithi087.stdout:Downloading Packages: 2024-04-02T15:16:56.517 INFO:teuthology.orchestra.run.smithi106.stdout:lab-extras 26 kB/s | 1.7 kB 00:00 2024-04-02T15:16:56.571 INFO:teuthology.orchestra.run.smithi087.stdout:(1/4): kernel-5.14.0-434.el9.x86_64.rpm 1.3 MB/s | 192 kB 00:00 2024-04-02T15:16:57.088 INFO:teuthology.orchestra.run.smithi087.stdout:(2/4): kernel-core-5.14.0-434.el9.x86_64.rpm 23 MB/s | 15 MB 00:00 2024-04-02T15:16:57.847 INFO:teuthology.orchestra.run.smithi087.stdout:(3/4): kernel-modules-5.14.0-434.el9.x86_64.rpm 24 MB/s | 34 MB 00:01 2024-04-02T15:16:58.097 INFO:teuthology.orchestra.run.smithi087.stdout:(4/4): kernel-modules-core-5.14.0-434.el9.x86_6 18 MB/s | 28 MB 00:01 2024-04-02T15:16:58.100 INFO:teuthology.orchestra.run.smithi087.stdout:-------------------------------------------------------------------------------- 2024-04-02T15:16:58.100 INFO:teuthology.orchestra.run.smithi087.stdout:Total 46 MB/s | 76 MB 00:01 2024-04-02T15:16:58.101 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction check 2024-04-02T15:16:58.449 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction check succeeded. 2024-04-02T15:16:58.449 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction test 2024-04-02T15:16:58.748 INFO:teuthology.orchestra.run.smithi106.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-02T15:16:58.748 INFO:teuthology.orchestra.run.smithi106.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-04-02T15:16:58.844 INFO:teuthology.orchestra.run.smithi106.stdout:Dependencies resolved. 2024-04-02T15:16:58.856 INFO:teuthology.orchestra.run.smithi106.stdout:================================================================================ 2024-04-02T15:16:58.856 INFO:teuthology.orchestra.run.smithi106.stdout: Package Arch Version Repository Size 2024-04-02T15:16:58.856 INFO:teuthology.orchestra.run.smithi106.stdout:================================================================================ 2024-04-02T15:16:58.856 INFO:teuthology.orchestra.run.smithi106.stdout:Installing: 2024-04-02T15:16:58.856 INFO:teuthology.orchestra.run.smithi106.stdout: kernel x86_64 5.14.0-434.el9 CentOS-BaseOS 192 k 2024-04-02T15:16:58.856 INFO:teuthology.orchestra.run.smithi106.stdout:Installing dependencies: 2024-04-02T15:16:58.856 INFO:teuthology.orchestra.run.smithi106.stdout: kernel-core x86_64 5.14.0-434.el9 CentOS-BaseOS 15 M 2024-04-02T15:16:58.856 INFO:teuthology.orchestra.run.smithi106.stdout: kernel-modules x86_64 5.14.0-434.el9 CentOS-BaseOS 34 M 2024-04-02T15:16:58.856 INFO:teuthology.orchestra.run.smithi106.stdout: kernel-modules-core x86_64 5.14.0-434.el9 CentOS-BaseOS 28 M 2024-04-02T15:16:58.856 INFO:teuthology.orchestra.run.smithi106.stdout: 2024-04-02T15:16:58.857 INFO:teuthology.orchestra.run.smithi106.stdout:Transaction Summary 2024-04-02T15:16:58.857 INFO:teuthology.orchestra.run.smithi106.stdout:================================================================================ 2024-04-02T15:16:58.857 INFO:teuthology.orchestra.run.smithi106.stdout:Install 4 Packages 2024-04-02T15:16:58.857 INFO:teuthology.orchestra.run.smithi106.stdout: 2024-04-02T15:16:58.858 INFO:teuthology.orchestra.run.smithi106.stdout:Total download size: 76 M 2024-04-02T15:16:58.858 INFO:teuthology.orchestra.run.smithi106.stdout:Installed size: 123 M 2024-04-02T15:16:58.858 INFO:teuthology.orchestra.run.smithi106.stdout:Downloading Packages: 2024-04-02T15:16:59.018 INFO:teuthology.orchestra.run.smithi106.stdout:(1/4): kernel-5.14.0-434.el9.x86_64.rpm 1.2 MB/s | 192 kB 00:00 2024-04-02T15:16:59.270 INFO:teuthology.orchestra.run.smithi087.stdout:Transaction test succeeded. 2024-04-02T15:16:59.271 INFO:teuthology.orchestra.run.smithi087.stdout:Running transaction 2024-04-02T15:16:59.693 INFO:teuthology.orchestra.run.smithi106.stdout:(2/4): kernel-core-5.14.0-434.el9.x86_64.rpm 18 MB/s | 15 MB 00:00 2024-04-02T15:17:00.377 INFO:teuthology.orchestra.run.smithi106.stdout:(3/4): kernel-modules-5.14.0-434.el9.x86_64.rpm 22 MB/s | 34 MB 00:01 2024-04-02T15:17:00.810 INFO:teuthology.orchestra.run.smithi106.stdout:(4/4): kernel-modules-core-5.14.0-434.el9.x86_6 15 MB/s | 28 MB 00:01 2024-04-02T15:17:00.813 INFO:teuthology.orchestra.run.smithi106.stdout:-------------------------------------------------------------------------------- 2024-04-02T15:17:00.813 INFO:teuthology.orchestra.run.smithi106.stdout:Total 39 MB/s | 76 MB 00:01 2024-04-02T15:17:00.814 INFO:teuthology.orchestra.run.smithi106.stdout:Running transaction check 2024-04-02T15:17:01.165 INFO:teuthology.orchestra.run.smithi106.stdout:Transaction check succeeded. 2024-04-02T15:17:01.166 INFO:teuthology.orchestra.run.smithi106.stdout:Running transaction test 2024-04-02T15:17:01.583 INFO:teuthology.orchestra.run.smithi087.stdout: Preparing : 1/1 2024-04-02T15:17:02.001 INFO:teuthology.orchestra.run.smithi106.stdout:Transaction test succeeded. 2024-04-02T15:17:02.001 INFO:teuthology.orchestra.run.smithi106.stdout:Running transaction 2024-04-02T15:17:02.177 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : kernel-modules-core-5.14.0-434.el9.x86_64 1/4 2024-04-02T15:17:02.264 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : kernel-core-5.14.0-434.el9.x86_64 2/4 2024-04-02T15:17:04.330 INFO:teuthology.orchestra.run.smithi106.stdout: Preparing : 1/1 2024-04-02T15:17:05.009 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: kernel-core-5.14.0-434.el9.x86_64 2/4 2024-04-02T15:17:05.110 INFO:teuthology.orchestra.run.smithi106.stdout: Installing : kernel-modules-core-5.14.0-434.el9.x86_64 1/4 2024-04-02T15:17:05.151 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-04-02T15:17:05.194 INFO:teuthology.orchestra.run.smithi106.stdout: Installing : kernel-core-5.14.0-434.el9.x86_64 2/4 2024-04-02T15:17:07.931 INFO:teuthology.orchestra.run.smithi106.stdout: Running scriptlet: kernel-core-5.14.0-434.el9.x86_64 2/4 2024-04-02T15:17:08.081 INFO:teuthology.orchestra.run.smithi106.stdout: Installing : kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-04-02T15:17:09.878 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-04-02T15:17:09.916 INFO:teuthology.orchestra.run.smithi087.stdout: Installing : kernel-5.14.0-434.el9.x86_64 4/4 2024-04-02T15:17:12.754 INFO:teuthology.orchestra.run.smithi106.stdout: Running scriptlet: kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-04-02T15:17:12.872 INFO:teuthology.orchestra.run.smithi106.stdout: Installing : kernel-5.14.0-434.el9.x86_64 4/4 2024-04-02T15:17:14.671 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: kernel-modules-core-5.14.0-434.el9.x86_64 4/4 2024-04-02T15:17:17.621 INFO:teuthology.orchestra.run.smithi106.stdout: Running scriptlet: kernel-modules-core-5.14.0-434.el9.x86_64 4/4 2024-04-02T15:17:43.902 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: kernel-core-5.14.0-434.el9.x86_64 4/4 2024-04-02T15:17:43.903 INFO:teuthology.orchestra.run.smithi087.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-04-02T15:17:43.903 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-02T15:17:43.944 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: kernel-modules-5.14.0-434.el9.x86_64 4/4 2024-04-02T15:17:44.307 INFO:teuthology.orchestra.run.smithi087.stdout: Running scriptlet: kernel-5.14.0-434.el9.x86_64 4/4 2024-04-02T15:17:44.307 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : kernel-5.14.0-434.el9.x86_64 1/4 2024-04-02T15:17:44.307 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : kernel-core-5.14.0-434.el9.x86_64 2/4 2024-04-02T15:17:44.307 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-04-02T15:17:44.711 INFO:teuthology.orchestra.run.smithi087.stdout: Verifying : kernel-modules-core-5.14.0-434.el9.x86_64 4/4 2024-04-02T15:17:44.711 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-02T15:17:44.711 INFO:teuthology.orchestra.run.smithi087.stdout:Installed: 2024-04-02T15:17:44.711 INFO:teuthology.orchestra.run.smithi087.stdout: kernel-5.14.0-434.el9.x86_64 2024-04-02T15:17:44.711 INFO:teuthology.orchestra.run.smithi087.stdout: kernel-core-5.14.0-434.el9.x86_64 2024-04-02T15:17:44.711 INFO:teuthology.orchestra.run.smithi087.stdout: kernel-modules-5.14.0-434.el9.x86_64 2024-04-02T15:17:44.711 INFO:teuthology.orchestra.run.smithi087.stdout: kernel-modules-core-5.14.0-434.el9.x86_64 2024-04-02T15:17:44.711 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-02T15:17:44.711 INFO:teuthology.orchestra.run.smithi087.stdout:Complete! 2024-04-02T15:17:45.037 DEBUG:teuthology.orchestra.run.smithi087:> rpm -q kernel | sort -rV | head -n 1 2024-04-02T15:17:45.141 INFO:teuthology.orchestra.run.smithi087.stdout:kernel-5.14.0-434.el9.x86_64 2024-04-02T15:17:45.142 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-434.el9.x86_64 2024-04-02T15:17:45.142 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-434.el9.x86_64 2024-04-02T15:17:45.142 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-02T15:17:45.142 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-02T15:17:45.142 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-04-02T15:17:45.142 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi087.front.sepia.ceph.com, path=None, version=distro) 2024-04-02T15:17:45.142 DEBUG:teuthology.orchestra.run.smithi087:> rpm -q kernel | sort -rV | head -n 1 2024-04-02T15:17:45.222 INFO:teuthology.orchestra.run.smithi087.stdout:kernel-5.14.0-434.el9.x86_64 2024-04-02T15:17:45.223 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-434.el9.x86_64 2024-04-02T15:17:45.223 DEBUG:teuthology.orchestra.run.smithi087:> sudo rpm -qi grub2-tools 2024-04-02T15:17:45.256 INFO:teuthology.orchestra.run.smithi087.stdout:Name : grub2-tools 2024-04-02T15:17:45.257 INFO:teuthology.orchestra.run.smithi087.stdout:Epoch : 1 2024-04-02T15:17:45.257 INFO:teuthology.orchestra.run.smithi087.stdout:Version : 2.06 2024-04-02T15:17:45.257 INFO:teuthology.orchestra.run.smithi087.stdout:Release : 68.el9 2024-04-02T15:17:45.257 INFO:teuthology.orchestra.run.smithi087.stdout:Architecture: x86_64 2024-04-02T15:17:45.257 INFO:teuthology.orchestra.run.smithi087.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-04-02T15:17:45.257 INFO:teuthology.orchestra.run.smithi087.stdout:Group : Unspecified 2024-04-02T15:17:45.257 INFO:teuthology.orchestra.run.smithi087.stdout:Size : 8264254 2024-04-02T15:17:45.257 INFO:teuthology.orchestra.run.smithi087.stdout:License : GPLv3+ 2024-04-02T15:17:45.257 INFO:teuthology.orchestra.run.smithi087.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-04-02T15:17:45.257 INFO:teuthology.orchestra.run.smithi087.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-04-02T15:17:45.257 INFO:teuthology.orchestra.run.smithi087.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-04-02T15:17:45.257 INFO:teuthology.orchestra.run.smithi087.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-04-02T15:17:45.257 INFO:teuthology.orchestra.run.smithi087.stdout:Packager : builder@centos.org 2024-04-02T15:17:45.257 INFO:teuthology.orchestra.run.smithi087.stdout:Vendor : CentOS 2024-04-02T15:17:45.258 INFO:teuthology.orchestra.run.smithi087.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-02T15:17:45.258 INFO:teuthology.orchestra.run.smithi087.stdout:Summary : Support tools for GRUB. 2024-04-02T15:17:45.258 INFO:teuthology.orchestra.run.smithi087.stdout:Description : 2024-04-02T15:17:45.258 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-02T15:17:45.258 INFO:teuthology.orchestra.run.smithi087.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-02T15:17:45.258 INFO:teuthology.orchestra.run.smithi087.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-02T15:17:45.258 INFO:teuthology.orchestra.run.smithi087.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-02T15:17:45.258 INFO:teuthology.orchestra.run.smithi087.stdout:hardware devices. 2024-04-02T15:17:45.258 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-02T15:17:45.258 INFO:teuthology.orchestra.run.smithi087.stdout:This subpackage provides tools for support of all platforms. 2024-04-02T15:17:45.259 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-02T15:17:45.259 INFO:teuthology.task.kernel:Updating grub on smithi087 to boot 5.14.0-434.el9.x86_64 2024-04-02T15:17:45.260 DEBUG:teuthology.orchestra.run.smithi087:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-02T15:17:45.924 INFO:teuthology.orchestra.run.smithi087.stderr:Generating grub configuration file ... 2024-04-02T15:17:46.994 INFO:teuthology.orchestra.run.smithi106.stdout: Running scriptlet: kernel-core-5.14.0-434.el9.x86_64 4/4 2024-04-02T15:17:46.994 INFO:teuthology.orchestra.run.smithi106.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-04-02T15:17:46.994 INFO:teuthology.orchestra.run.smithi106.stdout: 2024-04-02T15:17:47.038 INFO:teuthology.orchestra.run.smithi106.stdout: Running scriptlet: kernel-modules-5.14.0-434.el9.x86_64 4/4 2024-04-02T15:17:47.321 INFO:teuthology.orchestra.run.smithi087.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-04-02T15:17:47.356 INFO:teuthology.orchestra.run.smithi087.stderr:done 2024-04-02T15:17:47.358 DEBUG:teuthology.orchestra.run.smithi087:> mktemp 2024-04-02T15:17:47.372 INFO:teuthology.orchestra.run.smithi087.stdout:/tmp/tmp.dlYEVDpQPJ 2024-04-02T15:17:47.373 DEBUG:teuthology.orchestra.run.smithi087:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.dlYEVDpQPJ 2024-04-02T15:17:47.433 INFO:teuthology.orchestra.run.smithi106.stdout: Running scriptlet: kernel-5.14.0-434.el9.x86_64 4/4 2024-04-02T15:17:47.433 INFO:teuthology.orchestra.run.smithi106.stdout: Verifying : kernel-5.14.0-434.el9.x86_64 1/4 2024-04-02T15:17:47.433 INFO:teuthology.orchestra.run.smithi106.stdout: Verifying : kernel-core-5.14.0-434.el9.x86_64 2/4 2024-04-02T15:17:47.433 INFO:teuthology.orchestra.run.smithi106.stdout: Verifying : kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-04-02T15:17:47.442 DEBUG:teuthology.orchestra.run.smithi087:> sudo chmod 0666 /tmp/tmp.dlYEVDpQPJ 2024-04-02T15:17:47.588 DEBUG:teuthology.orchestra.remote:smithi087:/tmp/tmp.dlYEVDpQPJ is 6KB 2024-04-02T15:17:47.638 DEBUG:teuthology.orchestra.run.smithi087:> rm -fr /tmp/tmp.dlYEVDpQPJ 2024-04-02T15:17:47.653 DEBUG:teuthology.orchestra.run.smithi087:> sudo /bin/ls /boot/loader/entries || true 2024-04-02T15:17:47.721 INFO:teuthology.orchestra.run.smithi087.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-04-02T15:17:47.721 INFO:teuthology.orchestra.run.smithi087.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-04-02T15:17:47.721 INFO:teuthology.orchestra.run.smithi087.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-04-02T15:17:47.721 INFO:teuthology.orchestra.run.smithi087.stdout:15949e3c88704300882482eddfd7baae-5.14.0-434.el9.x86_64.conf 2024-04-02T15:17:47.722 DEBUG:teuthology.orchestra.run.smithi087:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-434.el9.x86_64 2024-04-02T15:17:47.874 INFO:teuthology.orchestra.run.smithi106.stdout: Verifying : kernel-modules-core-5.14.0-434.el9.x86_64 4/4 2024-04-02T15:17:47.874 INFO:teuthology.orchestra.run.smithi106.stdout: 2024-04-02T15:17:47.875 INFO:teuthology.orchestra.run.smithi106.stdout:Installed: 2024-04-02T15:17:47.875 INFO:teuthology.orchestra.run.smithi106.stdout: kernel-5.14.0-434.el9.x86_64 2024-04-02T15:17:47.875 INFO:teuthology.orchestra.run.smithi106.stdout: kernel-core-5.14.0-434.el9.x86_64 2024-04-02T15:17:47.875 INFO:teuthology.orchestra.run.smithi106.stdout: kernel-modules-5.14.0-434.el9.x86_64 2024-04-02T15:17:47.875 INFO:teuthology.orchestra.run.smithi106.stdout: kernel-modules-core-5.14.0-434.el9.x86_64 2024-04-02T15:17:47.875 INFO:teuthology.orchestra.run.smithi106.stdout: 2024-04-02T15:17:47.875 INFO:teuthology.orchestra.run.smithi106.stdout:Complete! 2024-04-02T15:17:47.876 DEBUG:teuthology.orchestra.run.smithi087:> sudo shutdown -r now 2024-04-02T15:17:48.196 DEBUG:teuthology.orchestra.run.smithi106:> rpm -q kernel | sort -rV | head -n 1 2024-04-02T15:17:48.340 INFO:teuthology.orchestra.run.smithi106.stdout:kernel-5.14.0-434.el9.x86_64 2024-04-02T15:17:48.340 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-434.el9.x86_64 2024-04-02T15:17:48.340 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-434.el9.x86_64 2024-04-02T15:17:48.340 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-02T15:17:48.340 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-02T15:17:48.340 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-04-02T15:17:48.340 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi106.front.sepia.ceph.com, path=None, version=distro) 2024-04-02T15:17:48.340 DEBUG:teuthology.orchestra.run.smithi106:> rpm -q kernel | sort -rV | head -n 1 2024-04-02T15:17:48.420 INFO:teuthology.orchestra.run.smithi106.stdout:kernel-5.14.0-434.el9.x86_64 2024-04-02T15:17:48.421 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-434.el9.x86_64 2024-04-02T15:17:48.421 DEBUG:teuthology.orchestra.run.smithi106:> sudo rpm -qi grub2-tools 2024-04-02T15:17:48.462 INFO:teuthology.orchestra.run.smithi106.stdout:Name : grub2-tools 2024-04-02T15:17:48.462 INFO:teuthology.orchestra.run.smithi106.stdout:Epoch : 1 2024-04-02T15:17:48.462 INFO:teuthology.orchestra.run.smithi106.stdout:Version : 2.06 2024-04-02T15:17:48.462 INFO:teuthology.orchestra.run.smithi106.stdout:Release : 68.el9 2024-04-02T15:17:48.462 INFO:teuthology.orchestra.run.smithi106.stdout:Architecture: x86_64 2024-04-02T15:17:48.462 INFO:teuthology.orchestra.run.smithi106.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-04-02T15:17:48.462 INFO:teuthology.orchestra.run.smithi106.stdout:Group : Unspecified 2024-04-02T15:17:48.463 INFO:teuthology.orchestra.run.smithi106.stdout:Size : 8264254 2024-04-02T15:17:48.463 INFO:teuthology.orchestra.run.smithi106.stdout:License : GPLv3+ 2024-04-02T15:17:48.463 INFO:teuthology.orchestra.run.smithi106.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-04-02T15:17:48.463 INFO:teuthology.orchestra.run.smithi106.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-04-02T15:17:48.463 INFO:teuthology.orchestra.run.smithi106.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-04-02T15:17:48.463 INFO:teuthology.orchestra.run.smithi106.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-04-02T15:17:48.463 INFO:teuthology.orchestra.run.smithi106.stdout:Packager : builder@centos.org 2024-04-02T15:17:48.463 INFO:teuthology.orchestra.run.smithi106.stdout:Vendor : CentOS 2024-04-02T15:17:48.463 INFO:teuthology.orchestra.run.smithi106.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-02T15:17:48.463 INFO:teuthology.orchestra.run.smithi106.stdout:Summary : Support tools for GRUB. 2024-04-02T15:17:48.464 INFO:teuthology.orchestra.run.smithi106.stdout:Description : 2024-04-02T15:17:48.464 INFO:teuthology.orchestra.run.smithi106.stdout: 2024-04-02T15:17:48.464 INFO:teuthology.orchestra.run.smithi106.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-02T15:17:48.464 INFO:teuthology.orchestra.run.smithi106.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-02T15:17:48.464 INFO:teuthology.orchestra.run.smithi106.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-02T15:17:48.464 INFO:teuthology.orchestra.run.smithi106.stdout:hardware devices. 2024-04-02T15:17:48.464 INFO:teuthology.orchestra.run.smithi106.stdout: 2024-04-02T15:17:48.464 INFO:teuthology.orchestra.run.smithi106.stdout:This subpackage provides tools for support of all platforms. 2024-04-02T15:17:48.465 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-02T15:17:48.465 INFO:teuthology.task.kernel:Updating grub on smithi106 to boot 5.14.0-434.el9.x86_64 2024-04-02T15:17:48.465 DEBUG:teuthology.orchestra.run.smithi106:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-02T15:17:49.075 INFO:teuthology.orchestra.run.smithi106.stderr:Generating grub configuration file ... 2024-04-02T15:17:50.463 INFO:teuthology.orchestra.run.smithi106.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-04-02T15:17:50.491 INFO:teuthology.orchestra.run.smithi106.stderr:done 2024-04-02T15:17:50.494 DEBUG:teuthology.orchestra.run.smithi106:> mktemp 2024-04-02T15:17:50.508 INFO:teuthology.orchestra.run.smithi106.stdout:/tmp/tmp.wyOYGXgikW 2024-04-02T15:17:50.509 DEBUG:teuthology.orchestra.run.smithi106:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.wyOYGXgikW 2024-04-02T15:17:50.577 DEBUG:teuthology.orchestra.run.smithi106:> sudo chmod 0666 /tmp/tmp.wyOYGXgikW 2024-04-02T15:17:50.716 DEBUG:teuthology.orchestra.remote:smithi106:/tmp/tmp.wyOYGXgikW is 6KB 2024-04-02T15:17:50.765 DEBUG:teuthology.orchestra.run.smithi106:> rm -fr /tmp/tmp.wyOYGXgikW 2024-04-02T15:17:50.779 DEBUG:teuthology.orchestra.run.smithi106:> sudo /bin/ls /boot/loader/entries || true 2024-04-02T15:17:50.844 INFO:teuthology.orchestra.run.smithi106.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-04-02T15:17:50.844 INFO:teuthology.orchestra.run.smithi106.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-04-02T15:17:50.844 INFO:teuthology.orchestra.run.smithi106.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-04-02T15:17:50.844 INFO:teuthology.orchestra.run.smithi106.stdout:15949e3c88704300882482eddfd7baae-5.14.0-434.el9.x86_64.conf 2024-04-02T15:17:50.846 DEBUG:teuthology.orchestra.run.smithi106:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-434.el9.x86_64 2024-04-02T15:17:50.997 DEBUG:teuthology.orchestra.run.smithi106:> sudo shutdown -r now 2024-04-02T15:18:17.905 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-04-02T15:18:17.905 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi087.front.sepia.ceph.com' 2024-04-02T15:18:17.906 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi087.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T15:18:21.002 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-04-02T15:18:21.002 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi106.front.sepia.ceph.com' 2024-04-02T15:18:21.003 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi106.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T15:18:36.323 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.87 2024-04-02T15:18:39.390 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.106 2024-04-02T15:18:45.328 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi087.front.sepia.ceph.com' 2024-04-02T15:18:45.329 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi087.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T15:18:48.382 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.87 2024-04-02T15:18:48.391 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi106.front.sepia.ceph.com' 2024-04-02T15:18:48.391 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi106.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T15:18:51.454 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.106 2024-04-02T15:19:00.390 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi087.front.sepia.ceph.com' 2024-04-02T15:19:00.391 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi087.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T15:19:03.458 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi106.front.sepia.ceph.com' 2024-04-02T15:19:03.459 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi106.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T15:19:21.887 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.106 2024-04-02T15:19:32.404 DEBUG:teuthology.orchestra.run.smithi087:> true 2024-04-02T15:19:32.788 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi087.front.sepia.ceph.com' 2024-04-02T15:19:32.789 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-434.el9.x86_64"... 2024-04-02T15:19:32.789 DEBUG:teuthology.orchestra.run.smithi087:> uname -r 2024-04-02T15:19:32.803 INFO:teuthology.orchestra.run.smithi087.stdout:5.14.0-434.el9.x86_64 2024-04-02T15:19:32.804 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-434.el9.x86_64 vs 5.14.0-434.el9.x86_64 2024-04-02T15:19:32.804 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-02T15:19:32.804 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-02T15:19:33.805 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-04-02T15:19:33.805 DEBUG:teuthology.orchestra.run.smithi087:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-02T15:19:33.884 INFO:teuthology.orchestra.run.smithi087.stdout:ttyS1 2024-04-02T15:19:33.926 DEBUG:teuthology.parallel:result is None 2024-04-02T15:19:36.888 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi106.front.sepia.ceph.com' 2024-04-02T15:19:36.889 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi106.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T15:19:37.224 DEBUG:teuthology.orchestra.run.smithi106:> true 2024-04-02T15:19:37.672 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi106.front.sepia.ceph.com' 2024-04-02T15:19:37.672 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-434.el9.x86_64"... 2024-04-02T15:19:37.672 DEBUG:teuthology.orchestra.run.smithi106:> uname -r 2024-04-02T15:19:37.687 INFO:teuthology.orchestra.run.smithi106.stdout:5.14.0-434.el9.x86_64 2024-04-02T15:19:37.688 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-434.el9.x86_64 vs 5.14.0-434.el9.x86_64 2024-04-02T15:19:37.688 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-02T15:19:37.688 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-02T15:19:38.689 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-04-02T15:19:38.689 DEBUG:teuthology.orchestra.run.smithi106:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-02T15:19:38.758 INFO:teuthology.orchestra.run.smithi106.stdout:ttyS1 2024-04-02T15:19:38.833 DEBUG:teuthology.parallel:result is None 2024-04-02T15:19:38.833 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-02T15:19:38.841 INFO:teuthology.task.internal:Creating test directory... 2024-04-02T15:19:38.841 DEBUG:teuthology.orchestra.run.smithi087:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-02T15:19:38.843 DEBUG:teuthology.orchestra.run.smithi106:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-02T15:19:38.858 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-02T15:19:38.900 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-02T15:19:38.906 INFO:teuthology.task.internal:Creating archive directory... 2024-04-02T15:19:38.906 DEBUG:teuthology.orchestra.run.smithi087:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-02T15:19:38.908 DEBUG:teuthology.orchestra.run.smithi106:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-02T15:19:38.941 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-02T15:19:38.948 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-02T15:19:38.948 DEBUG:teuthology.orchestra.run.smithi087:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-02T15:19:38.972 DEBUG:teuthology.orchestra.run.smithi106:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-02T15:19:39.000 INFO:teuthology.orchestra.run.smithi087.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-02T15:19:39.002 INFO:teuthology.orchestra.run.smithi106.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-02T15:19:39.014 INFO:teuthology.orchestra.run.smithi087.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-02T15:19:39.015 INFO:teuthology.orchestra.run.smithi106.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-02T15:19:39.017 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-02T15:19:39.024 INFO:teuthology.task.internal:Configuring sudo... 2024-04-02T15:19:39.024 DEBUG:teuthology.orchestra.run.smithi087:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-02T15:19:39.058 DEBUG:teuthology.orchestra.run.smithi106:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-02T15:19:39.086 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-02T15:19:39.095 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-02T15:19:39.095 DEBUG:teuthology.orchestra.run.smithi087:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-02T15:19:39.127 DEBUG:teuthology.orchestra.run.smithi106:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-02T15:19:39.142 DEBUG:teuthology.orchestra.run.smithi087:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-02T15:19:39.221 DEBUG:teuthology.orchestra.run.smithi087:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-02T15:19:39.286 DEBUG:teuthology.orchestra.run.smithi087:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-02T15:19:39.328 DEBUG:teuthology.orchestra.run.smithi087:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-02T15:19:39.370 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2024-04-02T15:19:39.370 DEBUG:teuthology.orchestra.run.smithi087:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-02T15:19:39.436 DEBUG:teuthology.orchestra.run.smithi106:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-02T15:19:39.480 DEBUG:teuthology.orchestra.run.smithi106:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-02T15:19:39.552 DEBUG:teuthology.orchestra.run.smithi106:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-02T15:19:39.592 DEBUG:teuthology.orchestra.run.smithi106:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-02T15:19:39.636 DEBUG:teuthology.orchestra.run.smithi106:> set -ex 2024-04-02T15:19:39.636 DEBUG:teuthology.orchestra.run.smithi106:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-02T15:19:39.703 DEBUG:teuthology.orchestra.run.smithi087:> sudo service rsyslog restart 2024-04-02T15:19:39.705 DEBUG:teuthology.orchestra.run.smithi106:> sudo service rsyslog restart 2024-04-02T15:19:39.746 INFO:teuthology.orchestra.run.smithi087.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-02T15:19:39.890 INFO:teuthology.orchestra.run.smithi106.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-02T15:19:40.145 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-02T15:19:40.153 INFO:teuthology.task.internal:Starting timer... 2024-04-02T15:19:40.153 INFO:teuthology.run_tasks:Running task pcp... 2024-04-02T15:19:40.187 INFO:teuthology.run_tasks:Running task selinux... 2024-04-02T15:19:40.196 DEBUG:teuthology.orchestra.run.smithi087:> sudo service auditd rotate 2024-04-02T15:19:40.253 INFO:teuthology.orchestra.run.smithi087.stdout:Rotating logs: 2024-04-02T15:19:40.255 DEBUG:teuthology.orchestra.run.smithi106:> sudo service auditd rotate 2024-04-02T15:19:40.351 INFO:teuthology.orchestra.run.smithi106.stdout:Rotating logs: 2024-04-02T15:19:40.354 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-02T15:19:40.354 DEBUG:teuthology.orchestra.run.smithi087:> /usr/sbin/getenforce 2024-04-02T15:19:40.381 INFO:teuthology.orchestra.run.smithi087.stdout:Permissive 2024-04-02T15:19:40.381 DEBUG:teuthology.orchestra.run.smithi106:> /usr/sbin/getenforce 2024-04-02T15:19:40.416 INFO:teuthology.orchestra.run.smithi106.stdout:Permissive 2024-04-02T15:19:40.416 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi087.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi106.front.sepia.ceph.com': 'permissive'} 2024-04-02T15:19:40.416 DEBUG:teuthology.orchestra.run.smithi087:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-02T15:19:40.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-02T15:19:40.448 DEBUG:teuthology.orchestra.run.smithi106:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-02T15:19:40.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-02T15:19:40.506 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-02T15:19:40.506 DEBUG:teuthology.orchestra.run.smithi087:> sudo /usr/sbin/setenforce permissive 2024-04-02T15:19:40.545 DEBUG:teuthology.orchestra.run.smithi106:> sudo /usr/sbin/setenforce permissive 2024-04-02T15:19:40.602 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-02T15:19:40.612 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-04-02T15:19:40.618 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-04-02T15:19:40.759 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-02T15:19:40.775 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-02T15:19:40.777 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi087.front.sepia.ceph.com,smithi106.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-02T15:23:52.903 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi087.front.sepia.ceph.com'), Remote(name='ubuntu@smithi106.front.sepia.ceph.com')] 2024-04-02T15:23:52.904 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi087.front.sepia.ceph.com' 2024-04-02T15:23:52.905 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi087.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T15:23:53.046 DEBUG:teuthology.orchestra.run.smithi087:> true 2024-04-02T15:23:53.127 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi087.front.sepia.ceph.com' 2024-04-02T15:23:53.128 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi106.front.sepia.ceph.com' 2024-04-02T15:23:53.128 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi106.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T15:23:53.226 DEBUG:teuthology.orchestra.run.smithi106:> true 2024-04-02T15:23:53.315 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi106.front.sepia.ceph.com' 2024-04-02T15:23:53.316 INFO:teuthology.run_tasks:Running task clock... 2024-04-02T15:23:53.629 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-04-02T15:23:53.629 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-02T15:23:53.630 DEBUG:teuthology.orchestra.run.smithi087:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-02T15:23:53.633 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-02T15:23:53.633 DEBUG:teuthology.orchestra.run.smithi106:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-02T15:23:53.667 INFO:teuthology.orchestra.run.smithi087.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-02T15:23:53.669 INFO:teuthology.orchestra.run.smithi106.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-02T15:23:53.685 INFO:teuthology.orchestra.run.smithi087.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-02T15:23:53.686 INFO:teuthology.orchestra.run.smithi106.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-02T15:23:53.716 INFO:teuthology.orchestra.run.smithi087.stderr:sudo: ntpd: command not found 2024-04-02T15:23:53.717 INFO:teuthology.orchestra.run.smithi106.stderr:sudo: ntpd: command not found 2024-04-02T15:23:53.730 INFO:teuthology.orchestra.run.smithi087.stdout:506 Cannot talk to daemon 2024-04-02T15:23:53.731 INFO:teuthology.orchestra.run.smithi106.stdout:506 Cannot talk to daemon 2024-04-02T15:23:53.747 INFO:teuthology.orchestra.run.smithi087.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-02T15:23:53.748 INFO:teuthology.orchestra.run.smithi106.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-02T15:23:53.764 INFO:teuthology.orchestra.run.smithi087.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-02T15:23:53.765 INFO:teuthology.orchestra.run.smithi106.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-02T15:23:53.816 INFO:teuthology.orchestra.run.smithi106.stderr:bash: line 1: ntpq: command not found 2024-04-02T15:23:53.818 INFO:teuthology.orchestra.run.smithi087.stderr:bash: line 1: ntpq: command not found 2024-04-02T15:23:53.819 INFO:teuthology.orchestra.run.smithi106.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-02T15:23:53.820 INFO:teuthology.orchestra.run.smithi106.stdout:=============================================================================== 2024-04-02T15:23:53.820 INFO:teuthology.orchestra.run.smithi106.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T15:23:53.820 INFO:teuthology.orchestra.run.smithi106.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T15:23:53.820 INFO:teuthology.orchestra.run.smithi106.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T15:23:53.820 INFO:teuthology.orchestra.run.smithi106.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T15:23:53.821 INFO:teuthology.orchestra.run.smithi087.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-02T15:23:53.821 INFO:teuthology.orchestra.run.smithi087.stdout:=============================================================================== 2024-04-02T15:23:53.821 INFO:teuthology.orchestra.run.smithi087.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T15:23:53.821 INFO:teuthology.orchestra.run.smithi087.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T15:23:53.821 INFO:teuthology.orchestra.run.smithi087.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T15:23:53.821 INFO:teuthology.orchestra.run.smithi087.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T15:23:53.823 INFO:teuthology.run_tasks:Running task cephadm... 2024-04-02T15:23:53.933 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1, 'mgr/cephadm/use_agent': True}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '3f3f3ed992429b85be729743ad3b39d52dd90744'} 2024-04-02T15:23:53.933 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-04-02T15:23:53.933 INFO:tasks.cephadm:Cluster fsid is 0400842c-f105-11ee-b647-cb9ed24678a4 2024-04-02T15:23:53.933 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-04-02T15:23:53.934 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.87', 'mon.c': '[v2:172.21.15.87:3301,v1:172.21.15.87:6790]', 'mon.b': '172.21.15.106'} 2024-04-02T15:23:53.934 INFO:tasks.cephadm:First mon is mon.a on smithi087 2024-04-02T15:23:53.934 INFO:tasks.cephadm:First mgr is y 2024-04-02T15:23:53.934 INFO:tasks.cephadm:Normalizing hostnames... 2024-04-02T15:23:53.934 DEBUG:teuthology.orchestra.run.smithi087:> sudo hostname $(hostname -s) 2024-04-02T15:23:53.962 DEBUG:teuthology.orchestra.run.smithi106:> sudo hostname $(hostname -s) 2024-04-02T15:23:53.990 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-04-02T15:23:53.990 DEBUG:teuthology.orchestra.run.smithi087:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-02T15:23:54.161 INFO:teuthology.orchestra.run.smithi087.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Apr 2 15:23 /home/ubuntu/cephtest/cephadm 2024-04-02T15:23:54.161 DEBUG:teuthology.orchestra.run.smithi106:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-02T15:23:54.322 INFO:teuthology.orchestra.run.smithi106.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Apr 2 15:23 /home/ubuntu/cephtest/cephadm 2024-04-02T15:23:54.322 DEBUG:teuthology.orchestra.run.smithi087:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-02T15:23:54.339 DEBUG:teuthology.orchestra.run.smithi106:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-02T15:23:54.364 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-04-02T15:23:54.364 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-04-02T15:23:54.382 DEBUG:teuthology.orchestra.run.smithi106:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-04-02T15:23:55.214 INFO:teuthology.orchestra.run.smithi106.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-02T15:23:55.280 INFO:teuthology.orchestra.run.smithi087.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-02T15:24:14.185 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-02T15:24:14.185 INFO:teuthology.orchestra.run.smithi087.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-04-02T15:24:14.185 INFO:teuthology.orchestra.run.smithi087.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-04-02T15:24:14.186 INFO:teuthology.orchestra.run.smithi087.stdout: "repo_digests": [ 2024-04-02T15:24:14.186 INFO:teuthology.orchestra.run.smithi087.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-04-02T15:24:14.186 INFO:teuthology.orchestra.run.smithi087.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-04-02T15:24:14.186 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-02T15:24:14.186 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-02T15:24:15.541 INFO:teuthology.orchestra.run.smithi106.stdout:{ 2024-04-02T15:24:15.541 INFO:teuthology.orchestra.run.smithi106.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-04-02T15:24:15.541 INFO:teuthology.orchestra.run.smithi106.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-04-02T15:24:15.541 INFO:teuthology.orchestra.run.smithi106.stdout: "repo_digests": [ 2024-04-02T15:24:15.541 INFO:teuthology.orchestra.run.smithi106.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-04-02T15:24:15.541 INFO:teuthology.orchestra.run.smithi106.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-04-02T15:24:15.541 INFO:teuthology.orchestra.run.smithi106.stdout: ] 2024-04-02T15:24:15.541 INFO:teuthology.orchestra.run.smithi106.stdout:} 2024-04-02T15:24:15.559 DEBUG:teuthology.orchestra.run.smithi087:> sudo mkdir -p /etc/ceph 2024-04-02T15:24:15.595 DEBUG:teuthology.orchestra.run.smithi106:> sudo mkdir -p /etc/ceph 2024-04-02T15:24:15.624 DEBUG:teuthology.orchestra.run.smithi087:> sudo chmod 777 /etc/ceph 2024-04-02T15:24:15.664 DEBUG:teuthology.orchestra.run.smithi106:> sudo chmod 777 /etc/ceph 2024-04-02T15:24:15.691 INFO:tasks.cephadm:Writing seed config... 2024-04-02T15:24:15.692 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2024-04-02T15:24:15.692 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-04-02T15:24:15.692 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-04-02T15:24:15.692 INFO:tasks.cephadm: override: [mgr] mgr/cephadm/use_agent = True 2024-04-02T15:24:15.692 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-04-02T15:24:15.692 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-04-02T15:24:15.692 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-04-02T15:24:15.692 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-04-02T15:24:15.692 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-04-02T15:24:15.693 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2024-04-02T15:24:15.693 DEBUG:teuthology.orchestra.run.smithi087:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-04-02T15:24:15.721 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 0400842c-f105-11ee-b647-cb9ed24678a4 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = true bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 mgr/cephadm/use_agent = True [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-04-02T15:24:15.721 DEBUG:teuthology.orchestra.run.smithi087:mon.a> sudo journalctl -f -n 0 -u ceph-0400842c-f105-11ee-b647-cb9ed24678a4@mon.a.service 2024-04-02T15:24:15.763 DEBUG:teuthology.orchestra.run.smithi087:mgr.y> sudo journalctl -f -n 0 -u ceph-0400842c-f105-11ee-b647-cb9ed24678a4@mgr.y.service 2024-04-02T15:24:15.806 INFO:tasks.cephadm:Bootstrapping... 2024-04-02T15:24:15.806 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.87 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-04-02T15:24:16.018 INFO:teuthology.orchestra.run.smithi087.stderr:-------------------------------------------------------------------------------- 2024-04-02T15:24:16.018 INFO:teuthology.orchestra.run.smithi087.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '0400842c-f105-11ee-b647-cb9ed24678a4', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.87', '--skip-admin-label'] 2024-04-02T15:24:16.042 INFO:teuthology.orchestra.run.smithi087.stderr:/bin/podman: 4.9.4-dev 2024-04-02T15:24:16.044 INFO:teuthology.orchestra.run.smithi087.stderr:Verifying podman|docker is present... 2024-04-02T15:24:16.065 INFO:teuthology.orchestra.run.smithi087.stderr:/bin/podman: 4.9.4-dev 2024-04-02T15:24:16.067 INFO:teuthology.orchestra.run.smithi087.stderr:Verifying lvm2 is present... 2024-04-02T15:24:16.067 INFO:teuthology.orchestra.run.smithi087.stderr:Verifying time synchronization is in place... 2024-04-02T15:24:16.075 INFO:teuthology.orchestra.run.smithi087.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-02T15:24:16.081 INFO:teuthology.orchestra.run.smithi087.stderr:systemctl: inactive 2024-04-02T15:24:16.090 INFO:teuthology.orchestra.run.smithi087.stderr:systemctl: enabled 2024-04-02T15:24:16.097 INFO:teuthology.orchestra.run.smithi087.stderr:systemctl: active 2024-04-02T15:24:16.098 INFO:teuthology.orchestra.run.smithi087.stderr:Unit chronyd.service is enabled and running 2024-04-02T15:24:16.098 INFO:teuthology.orchestra.run.smithi087.stderr:Repeating the final host check... 2024-04-02T15:24:16.119 INFO:teuthology.orchestra.run.smithi087.stderr:/bin/podman: 4.9.4-dev 2024-04-02T15:24:16.122 INFO:teuthology.orchestra.run.smithi087.stderr:podman (/bin/podman) version 4.9.4 is present 2024-04-02T15:24:16.122 INFO:teuthology.orchestra.run.smithi087.stderr:systemctl is present 2024-04-02T15:24:16.122 INFO:teuthology.orchestra.run.smithi087.stderr:lvcreate is present 2024-04-02T15:24:16.129 INFO:teuthology.orchestra.run.smithi087.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-02T15:24:16.136 INFO:teuthology.orchestra.run.smithi087.stderr:systemctl: inactive 2024-04-02T15:24:16.144 INFO:teuthology.orchestra.run.smithi087.stderr:systemctl: enabled 2024-04-02T15:24:16.152 INFO:teuthology.orchestra.run.smithi087.stderr:systemctl: active 2024-04-02T15:24:16.153 INFO:teuthology.orchestra.run.smithi087.stderr:Unit chronyd.service is enabled and running 2024-04-02T15:24:16.153 INFO:teuthology.orchestra.run.smithi087.stderr:Host looks OK 2024-04-02T15:24:16.153 INFO:teuthology.orchestra.run.smithi087.stderr:Cluster fsid: 0400842c-f105-11ee-b647-cb9ed24678a4 2024-04-02T15:24:16.153 INFO:teuthology.orchestra.run.smithi087.stderr:Acquiring lock 140024146492768 on /run/cephadm/0400842c-f105-11ee-b647-cb9ed24678a4.lock 2024-04-02T15:24:16.153 INFO:teuthology.orchestra.run.smithi087.stderr:Lock 140024146492768 acquired on /run/cephadm/0400842c-f105-11ee-b647-cb9ed24678a4.lock 2024-04-02T15:24:16.153 INFO:teuthology.orchestra.run.smithi087.stderr:Verifying IP 172.21.15.87 port 3300 ... 2024-04-02T15:24:16.154 INFO:teuthology.orchestra.run.smithi087.stderr:Verifying IP 172.21.15.87 port 6789 ... 2024-04-02T15:24:16.154 INFO:teuthology.orchestra.run.smithi087.stderr:Base mon IP is 172.21.15.87, final addrv is [v2:172.21.15.87:3300,v1:172.21.15.87:6789] 2024-04-02T15:24:16.156 INFO:teuthology.orchestra.run.smithi087.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.87 metric 100 2024-04-02T15:24:16.156 INFO:teuthology.orchestra.run.smithi087.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.87 metric 100 2024-04-02T15:24:16.159 INFO:teuthology.orchestra.run.smithi087.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-04-02T15:24:16.159 INFO:teuthology.orchestra.run.smithi087.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2024-04-02T15:24:16.159 INFO:teuthology.orchestra.run.smithi087.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 1024 expires 1796sec hoplimit 64 pref medium 2024-04-02T15:24:16.162 INFO:teuthology.orchestra.run.smithi087.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-04-02T15:24:16.162 INFO:teuthology.orchestra.run.smithi087.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-04-02T15:24:16.162 INFO:teuthology.orchestra.run.smithi087.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-02T15:24:16.162 INFO:teuthology.orchestra.run.smithi087.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-04-02T15:24:16.162 INFO:teuthology.orchestra.run.smithi087.stderr:/sbin/ip: inet6 fe80::ec4:7aff:fe8f:d5f5/64 scope link 2024-04-02T15:24:16.162 INFO:teuthology.orchestra.run.smithi087.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-02T15:24:16.163 INFO:teuthology.orchestra.run.smithi087.stderr:Mon IP `172.21.15.87` is in CIDR network `172.21.0.0/20` 2024-04-02T15:24:16.163 INFO:teuthology.orchestra.run.smithi087.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-04-02T15:24:16.163 INFO:teuthology.orchestra.run.smithi087.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-02T15:24:16.270 INFO:teuthology.orchestra.run.smithi087.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-04-02T15:24:16.667 INFO:teuthology.orchestra.run.smithi087.stderr:/bin/podman: Getting image source signatures 2024-04-02T15:24:16.667 INFO:teuthology.orchestra.run.smithi087.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-04-02T15:24:16.667 INFO:teuthology.orchestra.run.smithi087.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-04-02T15:24:16.667 INFO:teuthology.orchestra.run.smithi087.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-04-02T15:24:16.667 INFO:teuthology.orchestra.run.smithi087.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-04-02T15:24:16.668 INFO:teuthology.orchestra.run.smithi087.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-04-02T15:24:16.668 INFO:teuthology.orchestra.run.smithi087.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-04-02T15:24:16.788 INFO:teuthology.orchestra.run.smithi087.stderr:/bin/podman: Writing manifest to image destination 2024-04-02T15:24:16.891 INFO:teuthology.orchestra.run.smithi087.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-04-02T15:24:17.333 INFO:teuthology.orchestra.run.smithi087.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-04-02T15:24:17.856 INFO:teuthology.orchestra.run.smithi087.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-04-02T15:24:17.856 INFO:teuthology.orchestra.run.smithi087.stderr:Extracting ceph user uid/gid from container image... 2024-04-02T15:24:18.281 INFO:teuthology.orchestra.run.smithi087.stderr:stat: 167 167 2024-04-02T15:24:18.836 INFO:teuthology.orchestra.run.smithi087.stderr:Creating initial keys... 2024-04-02T15:24:19.236 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph-authtool: AQAjIwxmuwRlCRAAU+QCnpqIALle62CW+yD0pw== 2024-04-02T15:24:20.166 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph-authtool: AQAkIwxmvzwmBRAAsHDZe2e4wyjlI5QsFelrJg== 2024-04-02T15:24:21.063 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph-authtool: AQAlIwxmCAsaABAAu/CFwJ6tEApyOHM0akAlaQ== 2024-04-02T15:24:21.553 INFO:teuthology.orchestra.run.smithi087.stderr:Creating initial monmap... 2024-04-02T15:24:21.944 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-02T15:24:21.944 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-04-02T15:24:21.944 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 0400842c-f105-11ee-b647-cb9ed24678a4 2024-04-02T15:24:21.944 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-02T15:24:22.500 INFO:teuthology.orchestra.run.smithi087.stderr:monmaptool for a [v2:172.21.15.87:3300,v1:172.21.15.87:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-02T15:24:22.501 INFO:teuthology.orchestra.run.smithi087.stderr:setting min_mon_release = octopus 2024-04-02T15:24:22.501 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/monmaptool: set fsid to 0400842c-f105-11ee-b647-cb9ed24678a4 2024-04-02T15:24:22.501 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-02T15:24:22.501 INFO:teuthology.orchestra.run.smithi087.stderr: 2024-04-02T15:24:22.501 INFO:teuthology.orchestra.run.smithi087.stderr:Creating mon... 2024-04-02T15:24:23.373 INFO:teuthology.orchestra.run.smithi087.stderr:create mon.a on 2024-04-02T15:24:24.065 INFO:teuthology.orchestra.run.smithi087.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-04-02T15:24:24.065 INFO:teuthology.orchestra.run.smithi087.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-0400842c-f105-11ee-b647-cb9ed24678a4.target → /etc/systemd/system/ceph-0400842c-f105-11ee-b647-cb9ed24678a4.target. 2024-04-02T15:24:24.065 INFO:teuthology.orchestra.run.smithi087.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-0400842c-f105-11ee-b647-cb9ed24678a4.target → /etc/systemd/system/ceph-0400842c-f105-11ee-b647-cb9ed24678a4.target. 2024-04-02T15:24:24.065 INFO:teuthology.orchestra.run.smithi087.stderr:systemctl: Failed to reset failed state of unit ceph-0400842c-f105-11ee-b647-cb9ed24678a4@mon.a.service: Unit ceph-0400842c-f105-11ee-b647-cb9ed24678a4@mon.a.service not loaded. 2024-04-02T15:24:24.065 INFO:teuthology.orchestra.run.smithi087.stderr:systemctl: Created symlink /etc/systemd/system/ceph-0400842c-f105-11ee-b647-cb9ed24678a4.target.wants/ceph-0400842c-f105-11ee-b647-cb9ed24678a4@mon.a.service → /etc/systemd/system/ceph-0400842c-f105-11ee-b647-cb9ed24678a4@.service. 2024-04-02T15:24:24.699 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:24 smithi087 podman[24536]: 2024-04-02 15:24:24.664772344 +0000 UTC m=+0.345743805 container start 9c24ba7d9b3a3838691cb486e9cbcc3f4e3a11b650859b29e888efabcc493745 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mon-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, release=754, architecture=x86_64, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , RELEASE=HEAD, name=centos-stream, io.k8s.display-name=CentOS Stream 8, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream) 2024-04-02T15:24:24.699 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:24 smithi087 ceph-mon[24555]: mkfs 0400842c-f105-11ee-b647-cb9ed24678a4 2024-04-02T15:24:24.752 INFO:teuthology.orchestra.run.smithi087.stderr:systemctl: disabled 2024-04-02T15:24:24.758 INFO:teuthology.orchestra.run.smithi087.stderr:systemctl: inactive 2024-04-02T15:24:24.759 INFO:teuthology.orchestra.run.smithi087.stderr:firewalld.service is not enabled 2024-04-02T15:24:24.759 INFO:teuthology.orchestra.run.smithi087.stderr:Not possible to enable service . firewalld.service is not available 2024-04-02T15:24:24.759 INFO:teuthology.orchestra.run.smithi087.stderr:Waiting for mon to start... 2024-04-02T15:24:24.759 INFO:teuthology.orchestra.run.smithi087.stderr:Waiting for mon... 2024-04-02T15:24:24.970 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:24 smithi087 ceph-mon[24555]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-02T15:24:24.970 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:24 smithi087 bash[24536]: 9c24ba7d9b3a3838691cb486e9cbcc3f4e3a11b650859b29e888efabcc493745 2024-04-02T15:24:24.971 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:24 smithi087 systemd[1]: Started Ceph mon.a for 0400842c-f105-11ee-b647-cb9ed24678a4. 2024-04-02T15:24:25.245 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: cluster: 2024-04-02T15:24:25.245 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: id: 0400842c-f105-11ee-b647-cb9ed24678a4 2024-04-02T15:24:25.245 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-04-02T15:24:25.245 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: 2024-04-02T15:24:25.245 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: services: 2024-04-02T15:24:25.245 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: mon: 1 daemons, quorum a (age 0.548337s) 2024-04-02T15:24:25.245 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: mgr: no daemons active 2024-04-02T15:24:25.245 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-04-02T15:24:25.245 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: 2024-04-02T15:24:25.245 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: data: 2024-04-02T15:24:25.245 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-04-02T15:24:25.245 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-04-02T15:24:25.245 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-04-02T15:24:25.246 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: pgs: 2024-04-02T15:24:25.246 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: 2024-04-02T15:24:25.795 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:25 smithi087 ceph-mon[24555]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-02T15:24:25.795 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:25 smithi087 ceph-mon[24555]: monmap e1: 1 mons at {a=[v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0]} 2024-04-02T15:24:25.795 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:25 smithi087 ceph-mon[24555]: fsmap 2024-04-02T15:24:25.795 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:25 smithi087 ceph-mon[24555]: osdmap e1: 0 total, 0 up, 0 in 2024-04-02T15:24:25.795 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:25 smithi087 ceph-mon[24555]: mgrmap e1: no daemons active 2024-04-02T15:24:25.795 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:25 smithi087 ceph-mon[24555]: from='client.? 172.21.15.87:0/3082375870' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-02T15:24:25.797 INFO:teuthology.orchestra.run.smithi087.stderr:mon is available 2024-04-02T15:24:25.798 INFO:teuthology.orchestra.run.smithi087.stderr:Assimilating anything we can from ceph.conf... 2024-04-02T15:24:26.312 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: 2024-04-02T15:24:26.312 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: [global] 2024-04-02T15:24:26.312 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: fsid = 0400842c-f105-11ee-b647-cb9ed24678a4 2024-04-02T15:24:26.312 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.87:3300,v1:172.21.15.87:6789] 2024-04-02T15:24:26.312 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-04-02T15:24:26.312 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-04-02T15:24:26.312 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-04-02T15:24:26.312 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-04-02T15:24:26.313 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: 2024-04-02T15:24:26.313 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: [mgr] 2024-04-02T15:24:26.313 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: mgr/cephadm/use_agent = True 2024-04-02T15:24:26.313 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-04-02T15:24:26.313 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: 2024-04-02T15:24:26.313 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: [osd] 2024-04-02T15:24:26.313 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-04-02T15:24:26.313 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-04-02T15:24:26.817 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:26 smithi087 ceph-mon[24555]: from='client.? 172.21.15.87:0/455800323' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-02T15:24:26.817 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:26 smithi087 ceph-mon[24555]: from='client.? 172.21.15.87:0/455800323' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-02T15:24:26.820 INFO:teuthology.orchestra.run.smithi087.stderr:Generating new minimal ceph.conf... 2024-04-02T15:24:27.865 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:27 smithi087 ceph-mon[24555]: from='client.? 172.21.15.87:0/3251830619' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:24:27.869 INFO:teuthology.orchestra.run.smithi087.stderr:Restarting the monitor... 2024-04-02T15:24:28.118 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:27 smithi087 systemd[1]: Stopping Ceph mon.a for 0400842c-f105-11ee-b647-cb9ed24678a4... 2024-04-02T15:24:28.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:27 smithi087 bash[24771]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mon.a" found: no such container 2024-04-02T15:24:28.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:28 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mon-a[24551]: 2024-04-02T15:24:28.023+0000 7f83454e6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T15:24:28.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:28 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mon-a[24551]: 2024-04-02T15:24:28.023+0000 7f83454e6700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-04-02T15:24:28.509 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:28 smithi087 podman[24782]: 2024-04-02 15:24:28.218420106 +0000 UTC m=+0.287242051 container died 9c24ba7d9b3a3838691cb486e9cbcc3f4e3a11b650859b29e888efabcc493745 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mon-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, distribution-scope=public, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, name=centos-stream, vcs-type=git) 2024-04-02T15:24:28.509 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:28 smithi087 podman[24782]: 2024-04-02 15:24:28.514921159 +0000 UTC m=+0.583743102 container cleanup 9c24ba7d9b3a3838691cb486e9cbcc3f4e3a11b650859b29e888efabcc493745 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mon-a, vendor=Red Hat, Inc., architecture=x86_64, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , RELEASE=HEAD, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, version=8) 2024-04-02T15:24:28.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:28 smithi087 bash[24782]: ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mon-a 2024-04-02T15:24:28.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:28 smithi087 bash[24808]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mon.a" found: no such container 2024-04-02T15:24:28.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:28 smithi087 podman[24796]: 2024-04-02 15:24:28.690698819 +0000 UTC m=+0.467364646 container remove 9c24ba7d9b3a3838691cb486e9cbcc3f4e3a11b650859b29e888efabcc493745 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mon-a, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , RELEASE=HEAD, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, GIT_BRANCH=HEAD, GIT_CLEAN=True, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, version=8, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8) 2024-04-02T15:24:28.872 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:28 smithi087 systemd[1]: ceph-0400842c-f105-11ee-b647-cb9ed24678a4@mon.a.service: Deactivated successfully. 2024-04-02T15:24:28.873 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:28 smithi087 systemd[1]: Stopped Ceph mon.a for 0400842c-f105-11ee-b647-cb9ed24678a4. 2024-04-02T15:24:28.873 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:28 smithi087 systemd[1]: Starting Ceph mon.a for 0400842c-f105-11ee-b647-cb9ed24678a4... 2024-04-02T15:24:29.195 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:28 smithi087 podman[24879]: 2024-04-02 15:24:28.834829016 +0000 UTC m=+0.018342089 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-04-02T15:24:29.195 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:28 smithi087 podman[24879]: 2024-04-02 15:24:28.933224986 +0000 UTC m=+0.116738085 container create e28fd32a7d9a16e9cd29d93249526e5ab8e7c89a1154253f4f465df687030731 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mon-a, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, build-date=2022-05-03T08:36:31.336870, ceph=True, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., RELEASE=HEAD, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, name=centos-stream, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , release=754, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container) 2024-04-02T15:24:29.195 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 podman[24879]: 2024-04-02 15:24:29.110186634 +0000 UTC m=+0.293699703 container init e28fd32a7d9a16e9cd29d93249526e5ab8e7c89a1154253f4f465df687030731 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mon-a, io.openshift.expose-services=, vcs-type=git, version=8, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, ceph=True, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-02T15:24:29.195 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 podman[24879]: 2024-04-02 15:24:29.113009323 +0000 UTC m=+0.296522377 container start e28fd32a7d9a16e9cd29d93249526e5ab8e7c89a1154253f4f465df687030731 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mon-a, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, distribution-scope=public, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, name=centos-stream, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, ceph=True, architecture=x86_64, GIT_CLEAN=True, vcs-type=git, io.openshift.expose-services=, vendor=Red Hat, Inc., RELEASE=HEAD, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux ) 2024-04-02T15:24:29.195 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: set uid:gid to 167:167 (ceph:ceph) 2024-04-02T15:24:29.195 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-04-02T15:24:29.195 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: pidfile_write: ignore empty --pid-file 2024-04-02T15:24:29.196 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: load: jerasure load: lrc 2024-04-02T15:24:29.196 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: RocksDB version: 6.15.5 2024-04-02T15:24:29.196 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-02T15:24:29.196 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Compile date Apr 18 2022 2024-04-02T15:24:29.196 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: DB SUMMARY 2024-04-02T15:24:29.196 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: DB Session ID: BKQX68JU01QX987EYJ3G 2024-04-02T15:24:29.196 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: CURRENT file: CURRENT 2024-04-02T15:24:29.196 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: IDENTITY file: IDENTITY 2024-04-02T15:24:29.196 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-04-02T15:24:29.196 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-04-02T15:24:29.196 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 82629 ; 2024-04-02T15:24:29.196 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.error_if_exists: 0 2024-04-02T15:24:29.196 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.create_if_missing: 0 2024-04-02T15:24:29.197 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.paranoid_checks: 1 2024-04-02T15:24:29.197 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-02T15:24:29.197 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.env: 0x5568d1ec6860 2024-04-02T15:24:29.197 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.fs: Posix File System 2024-04-02T15:24:29.197 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.info_log: 0x5568d2783dc0 2024-04-02T15:24:29.197 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.max_file_opening_threads: 16 2024-04-02T15:24:29.198 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.statistics: (nil) 2024-04-02T15:24:29.198 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.use_fsync: 0 2024-04-02T15:24:29.198 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.max_log_file_size: 0 2024-04-02T15:24:29.198 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-02T15:24:29.198 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-02T15:24:29.198 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.keep_log_file_num: 1000 2024-04-02T15:24:29.198 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.recycle_log_file_num: 0 2024-04-02T15:24:29.198 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.allow_fallocate: 1 2024-04-02T15:24:29.198 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.allow_mmap_reads: 0 2024-04-02T15:24:29.198 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.allow_mmap_writes: 0 2024-04-02T15:24:29.198 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.use_direct_reads: 0 2024-04-02T15:24:29.199 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-02T15:24:29.199 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.create_missing_column_families: 0 2024-04-02T15:24:29.199 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.db_log_dir: 2024-04-02T15:24:29.199 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-04-02T15:24:29.199 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-02T15:24:29.199 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-02T15:24:29.199 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-02T15:24:29.199 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-02T15:24:29.199 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-02T15:24:29.199 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-02T15:24:29.199 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.advise_random_on_open: 1 2024-04-02T15:24:29.199 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.db_write_buffer_size: 0 2024-04-02T15:24:29.199 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.write_buffer_manager: 0x5568d28742a0 2024-04-02T15:24:29.199 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-02T15:24:29.200 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-02T15:24:29.200 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-02T15:24:29.200 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-02T15:24:29.200 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.rate_limiter: (nil) 2024-04-02T15:24:29.200 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-02T15:24:29.200 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.wal_recovery_mode: 2 2024-04-02T15:24:29.200 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.enable_thread_tracking: 0 2024-04-02T15:24:29.200 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.enable_pipelined_write: 0 2024-04-02T15:24:29.200 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.unordered_write: 0 2024-04-02T15:24:29.200 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-02T15:24:29.200 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-02T15:24:29.200 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-02T15:24:29.200 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-02T15:24:29.200 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.row_cache: None 2024-04-02T15:24:29.201 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.wal_filter: None 2024-04-02T15:24:29.201 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-02T15:24:29.201 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.allow_ingest_behind: 0 2024-04-02T15:24:29.201 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.preserve_deletes: 0 2024-04-02T15:24:29.203 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.two_write_queues: 0 2024-04-02T15:24:29.203 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.manual_wal_flush: 0 2024-04-02T15:24:29.203 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.atomic_flush: 0 2024-04-02T15:24:29.203 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-02T15:24:29.203 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-02T15:24:29.203 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-02T15:24:29.203 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.log_readahead_size: 0 2024-04-02T15:24:29.203 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-02T15:24:29.203 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.best_efforts_recovery: 0 2024-04-02T15:24:29.203 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-02T15:24:29.203 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-02T15:24:29.204 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.allow_data_in_errors: 0 2024-04-02T15:24:29.204 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.db_host_id: __hostname__ 2024-04-02T15:24:29.204 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.max_background_jobs: 2 2024-04-02T15:24:29.204 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.max_background_compactions: -1 2024-04-02T15:24:29.204 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.max_subcompactions: 1 2024-04-02T15:24:29.204 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-02T15:24:29.204 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-02T15:24:29.204 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-02T15:24:29.204 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.max_total_wal_size: 0 2024-04-02T15:24:29.205 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-02T15:24:29.205 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-02T15:24:29.205 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-02T15:24:29.205 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-02T15:24:29.205 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.max_open_files: -1 2024-04-02T15:24:29.205 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.bytes_per_sync: 0 2024-04-02T15:24:29.205 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-02T15:24:29.205 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-02T15:24:29.205 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.compaction_readahead_size: 0 2024-04-02T15:24:29.205 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.max_background_flushes: -1 2024-04-02T15:24:29.205 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Compression algorithms supported: 2024-04-02T15:24:29.205 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-02T15:24:29.205 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: kZSTD supported: 0 2024-04-02T15:24:29.206 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: kXpressCompression supported: 0 2024-04-02T15:24:29.206 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: kLZ4HCCompression supported: 1 2024-04-02T15:24:29.206 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: kLZ4Compression supported: 1 2024-04-02T15:24:29.206 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: kBZip2Compression supported: 0 2024-04-02T15:24:29.206 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: kZlibCompression supported: 1 2024-04-02T15:24:29.206 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: kSnappyCompression supported: 1 2024-04-02T15:24:29.206 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-02T15:24:29.206 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-04-02T15:24:29.206 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-04-02T15:24:29.207 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-02T15:24:29.207 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.merge_operator: 2024-04-02T15:24:29.207 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.compaction_filter: None 2024-04-02T15:24:29.207 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.compaction_filter_factory: None 2024-04-02T15:24:29.207 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.sst_partitioner_factory: None 2024-04-02T15:24:29.207 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-02T15:24:29.207 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-02T15:24:29.207 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5568d274fd00) 2024-04-02T15:24:29.207 INFO:journalctl@ceph.mon.a.smithi087.stdout: cache_index_and_filter_blocks: 1 2024-04-02T15:24:29.207 INFO:journalctl@ceph.mon.a.smithi087.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-02T15:24:29.207 INFO:journalctl@ceph.mon.a.smithi087.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-02T15:24:29.207 INFO:journalctl@ceph.mon.a.smithi087.stdout: pin_top_level_index_and_filter: 1 2024-04-02T15:24:29.208 INFO:journalctl@ceph.mon.a.smithi087.stdout: index_type: 0 2024-04-02T15:24:29.208 INFO:journalctl@ceph.mon.a.smithi087.stdout: data_block_index_type: 0 2024-04-02T15:24:29.208 INFO:journalctl@ceph.mon.a.smithi087.stdout: index_shortening: 1 2024-04-02T15:24:29.208 INFO:journalctl@ceph.mon.a.smithi087.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-02T15:24:29.208 INFO:journalctl@ceph.mon.a.smithi087.stdout: hash_index_allow_collision: 1 2024-04-02T15:24:29.208 INFO:journalctl@ceph.mon.a.smithi087.stdout: checksum: 1 2024-04-02T15:24:29.208 INFO:journalctl@ceph.mon.a.smithi087.stdout: no_block_cache: 0 2024-04-02T15:24:29.208 INFO:journalctl@ceph.mon.a.smithi087.stdout: block_cache: 0x5568d27ba170 2024-04-02T15:24:29.208 INFO:journalctl@ceph.mon.a.smithi087.stdout: block_cache_name: BinnedLRUCache 2024-04-02T15:24:29.209 INFO:journalctl@ceph.mon.a.smithi087.stdout: block_cache_options: 2024-04-02T15:24:29.209 INFO:journalctl@ceph.mon.a.smithi087.stdout: capacity : 536870912 2024-04-02T15:24:29.209 INFO:journalctl@ceph.mon.a.smithi087.stdout: num_shard_bits : 4 2024-04-02T15:24:29.209 INFO:journalctl@ceph.mon.a.smithi087.stdout: strict_capacity_limit : 0 2024-04-02T15:24:29.209 INFO:journalctl@ceph.mon.a.smithi087.stdout: high_pri_pool_ratio: 0.000 2024-04-02T15:24:29.209 INFO:journalctl@ceph.mon.a.smithi087.stdout: block_cache_compressed: (nil) 2024-04-02T15:24:29.209 INFO:journalctl@ceph.mon.a.smithi087.stdout: persistent_cache: (nil) 2024-04-02T15:24:29.209 INFO:journalctl@ceph.mon.a.smithi087.stdout: block_size: 4096 2024-04-02T15:24:29.209 INFO:journalctl@ceph.mon.a.smithi087.stdout: block_size_deviation: 10 2024-04-02T15:24:29.209 INFO:journalctl@ceph.mon.a.smithi087.stdout: block_restart_interval: 16 2024-04-02T15:24:29.209 INFO:journalctl@ceph.mon.a.smithi087.stdout: index_block_restart_interval: 1 2024-04-02T15:24:29.209 INFO:journalctl@ceph.mon.a.smithi087.stdout: metadata_block_size: 4096 2024-04-02T15:24:29.210 INFO:journalctl@ceph.mon.a.smithi087.stdout: partition_filters: 0 2024-04-02T15:24:29.210 INFO:journalctl@ceph.mon.a.smithi087.stdout: use_delta_encoding: 1 2024-04-02T15:24:29.210 INFO:journalctl@ceph.mon.a.smithi087.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-02T15:24:29.210 INFO:journalctl@ceph.mon.a.smithi087.stdout: whole_key_filtering: 1 2024-04-02T15:24:29.210 INFO:journalctl@ceph.mon.a.smithi087.stdout: verify_compression: 0 2024-04-02T15:24:29.210 INFO:journalctl@ceph.mon.a.smithi087.stdout: read_amp_bytes_per_bit: 0 2024-04-02T15:24:29.210 INFO:journalctl@ceph.mon.a.smithi087.stdout: format_version: 4 2024-04-02T15:24:29.210 INFO:journalctl@ceph.mon.a.smithi087.stdout: enable_index_compression: 1 2024-04-02T15:24:29.210 INFO:journalctl@ceph.mon.a.smithi087.stdout: block_align: 0 2024-04-02T15:24:29.210 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.write_buffer_size: 33554432 2024-04-02T15:24:29.211 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.max_write_buffer_number: 2 2024-04-02T15:24:29.211 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.compression: NoCompression 2024-04-02T15:24:29.211 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.bottommost_compression: Disabled 2024-04-02T15:24:29.211 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.prefix_extractor: nullptr 2024-04-02T15:24:29.211 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-02T15:24:29.211 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.num_levels: 7 2024-04-02T15:24:29.211 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-02T15:24:29.211 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-02T15:24:29.211 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-02T15:24:29.212 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-02T15:24:29.212 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-02T15:24:29.212 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-02T15:24:29.212 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-02T15:24:29.212 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-02T15:24:29.212 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-02T15:24:29.212 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-02T15:24:29.212 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-02T15:24:29.212 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.compression_opts.level: 32767 2024-04-02T15:24:29.212 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.compression_opts.strategy: 0 2024-04-02T15:24:29.212 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-02T15:24:29.212 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-02T15:24:29.213 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-02T15:24:29.213 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.compression_opts.enabled: false 2024-04-02T15:24:29.213 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-02T15:24:29.213 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-02T15:24:29.213 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-02T15:24:29.213 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.target_file_size_base: 67108864 2024-04-02T15:24:29.213 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-02T15:24:29.213 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-02T15:24:29.213 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-02T15:24:29.213 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-02T15:24:29.213 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-02T15:24:29.213 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-02T15:24:29.214 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-02T15:24:29.214 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-02T15:24:29.214 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-02T15:24:29.214 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-02T15:24:29.214 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-02T15:24:29.214 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-02T15:24:29.214 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-02T15:24:29.214 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.arena_block_size: 4194304 2024-04-02T15:24:29.214 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-02T15:24:29.214 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-02T15:24:29.214 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-02T15:24:29.214 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.disable_auto_compactions: 0 2024-04-02T15:24:29.215 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-02T15:24:29.215 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-02T15:24:29.215 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-02T15:24:29.215 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-02T15:24:29.215 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-02T15:24:29.215 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-02T15:24:29.215 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-02T15:24:29.215 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-02T15:24:29.215 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-02T15:24:29.215 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-02T15:24:29.215 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.table_properties_collectors: 2024-04-02T15:24:29.215 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.inplace_update_support: 0 2024-04-02T15:24:29.216 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-02T15:24:29.216 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-02T15:24:29.216 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-02T15:24:29.216 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-02T15:24:29.216 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.bloom_locality: 0 2024-04-02T15:24:29.216 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.max_successive_merges: 0 2024-04-02T15:24:29.216 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-02T15:24:29.216 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.paranoid_file_checks: 0 2024-04-02T15:24:29.216 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.force_consistency_checks: 1 2024-04-02T15:24:29.216 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.report_bg_io_stats: 0 2024-04-02T15:24:29.217 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.ttl: 2592000 2024-04-02T15:24:29.217 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-02T15:24:29.217 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.enable_blob_files: false 2024-04-02T15:24:29.217 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.min_blob_size: 0 2024-04-02T15:24:29.217 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.blob_file_size: 268435456 2024-04-02T15:24:29.217 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-02T15:24:29.217 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-02T15:24:29.217 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-02T15:24:29.217 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-02T15:24:29.217 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-04-02T15:24:29.217 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-04-02T15:24:29.217 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712071469137819, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-04-02T15:24:29.217 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-04-02T15:24:29.218 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-02T15:24:29.218 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712071469138509, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79179, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77442, "index_size": 225, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10027, "raw_average_key_size": 47, "raw_value_size": 71751, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 211, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1712071469, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "dd2bddcb-bfe2-4c0a-9a17-a3b7964fe624", "db_session_id": "BKQX68JU01QX987EYJ3G"}} 2024-04-02T15:24:29.218 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-04-02T15:24:29.218 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712071469138825, "job": 1, "event": "recovery_finished"} 2024-04-02T15:24:29.218 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-02T15:24:29.218 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x5568d27a0a80 2024-04-02T15:24:29.218 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: DB pointer 0x5568d27b0000 2024-04-02T15:24:29.218 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-04-02T15:24:29.218 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-02T15:24:29.218 INFO:journalctl@ceph.mon.a.smithi087.stdout: ** DB Stats ** 2024-04-02T15:24:29.219 INFO:journalctl@ceph.mon.a.smithi087.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-02T15:24:29.219 INFO:journalctl@ceph.mon.a.smithi087.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-02T15:24:29.219 INFO:journalctl@ceph.mon.a.smithi087.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-02T15:24:29.219 INFO:journalctl@ceph.mon.a.smithi087.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-02T15:24:29.219 INFO:journalctl@ceph.mon.a.smithi087.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-02T15:24:29.219 INFO:journalctl@ceph.mon.a.smithi087.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-02T15:24:29.219 INFO:journalctl@ceph.mon.a.smithi087.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-02T15:24:29.219 INFO:journalctl@ceph.mon.a.smithi087.stdout: 2024-04-02T15:24:29.219 INFO:journalctl@ceph.mon.a.smithi087.stdout: ** Compaction Stats [default] ** 2024-04-02T15:24:29.219 INFO:journalctl@ceph.mon.a.smithi087.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-02T15:24:29.219 INFO:journalctl@ceph.mon.a.smithi087.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-02T15:24:29.219 INFO:journalctl@ceph.mon.a.smithi087.stdout: L0 2/0 79.08 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 167.1 0.00 0.00 1 0.000 0 0 2024-04-02T15:24:29.219 INFO:journalctl@ceph.mon.a.smithi087.stdout: Sum 2/0 79.08 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 167.1 0.00 0.00 1 0.000 0 0 2024-04-02T15:24:29.220 INFO:journalctl@ceph.mon.a.smithi087.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 167.1 0.00 0.00 1 0.000 0 0 2024-04-02T15:24:29.220 INFO:journalctl@ceph.mon.a.smithi087.stdout: 2024-04-02T15:24:29.220 INFO:journalctl@ceph.mon.a.smithi087.stdout: ** Compaction Stats [default] ** 2024-04-02T15:24:29.220 INFO:journalctl@ceph.mon.a.smithi087.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-02T15:24:29.220 INFO:journalctl@ceph.mon.a.smithi087.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-02T15:24:29.220 INFO:journalctl@ceph.mon.a.smithi087.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 167.1 0.00 0.00 1 0.000 0 0 2024-04-02T15:24:29.220 INFO:journalctl@ceph.mon.a.smithi087.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-02T15:24:29.220 INFO:journalctl@ceph.mon.a.smithi087.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-02T15:24:29.220 INFO:journalctl@ceph.mon.a.smithi087.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-02T15:24:29.220 INFO:journalctl@ceph.mon.a.smithi087.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-02T15:24:29.220 INFO:journalctl@ceph.mon.a.smithi087.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-02T15:24:29.220 INFO:journalctl@ceph.mon.a.smithi087.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-02T15:24:29.221 INFO:journalctl@ceph.mon.a.smithi087.stdout: Cumulative compaction: 0.00 GB write, 26.72 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T15:24:29.221 INFO:journalctl@ceph.mon.a.smithi087.stdout: Interval compaction: 0.00 GB write, 26.72 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T15:24:29.221 INFO:journalctl@ceph.mon.a.smithi087.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-02T15:24:29.221 INFO:journalctl@ceph.mon.a.smithi087.stdout: 2024-04-02T15:24:29.221 INFO:journalctl@ceph.mon.a.smithi087.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-02T15:24:29.221 INFO:journalctl@ceph.mon.a.smithi087.stdout: 2024-04-02T15:24:29.221 INFO:journalctl@ceph.mon.a.smithi087.stdout: ** Compaction Stats [default] ** 2024-04-02T15:24:29.221 INFO:journalctl@ceph.mon.a.smithi087.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-02T15:24:29.221 INFO:journalctl@ceph.mon.a.smithi087.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-02T15:24:29.221 INFO:journalctl@ceph.mon.a.smithi087.stdout: L0 2/0 79.08 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 167.1 0.00 0.00 1 0.000 0 0 2024-04-02T15:24:29.222 INFO:journalctl@ceph.mon.a.smithi087.stdout: Sum 2/0 79.08 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 167.1 0.00 0.00 1 0.000 0 0 2024-04-02T15:24:29.222 INFO:journalctl@ceph.mon.a.smithi087.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-02T15:24:29.222 INFO:journalctl@ceph.mon.a.smithi087.stdout: 2024-04-02T15:24:29.222 INFO:journalctl@ceph.mon.a.smithi087.stdout: ** Compaction Stats [default] ** 2024-04-02T15:24:29.222 INFO:journalctl@ceph.mon.a.smithi087.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-02T15:24:29.222 INFO:journalctl@ceph.mon.a.smithi087.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-02T15:24:29.222 INFO:journalctl@ceph.mon.a.smithi087.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 167.1 0.00 0.00 1 0.000 0 0 2024-04-02T15:24:29.222 INFO:journalctl@ceph.mon.a.smithi087.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-02T15:24:29.222 INFO:journalctl@ceph.mon.a.smithi087.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-02T15:24:29.222 INFO:journalctl@ceph.mon.a.smithi087.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-02T15:24:29.222 INFO:journalctl@ceph.mon.a.smithi087.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-02T15:24:29.222 INFO:journalctl@ceph.mon.a.smithi087.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-02T15:24:29.223 INFO:journalctl@ceph.mon.a.smithi087.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-02T15:24:29.223 INFO:journalctl@ceph.mon.a.smithi087.stdout: Cumulative compaction: 0.00 GB write, 26.41 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T15:24:29.223 INFO:journalctl@ceph.mon.a.smithi087.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T15:24:29.223 INFO:journalctl@ceph.mon.a.smithi087.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-02T15:24:29.223 INFO:journalctl@ceph.mon.a.smithi087.stdout: 2024-04-02T15:24:29.223 INFO:journalctl@ceph.mon.a.smithi087.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-02T15:24:29.223 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: starting mon.a rank 0 at public addrs [v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0] at bind addrs [v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 0400842c-f105-11ee-b647-cb9ed24678a4 2024-04-02T15:24:29.223 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: mon.a@-1(???) e1 preinit fsid 0400842c-f105-11ee-b647-cb9ed24678a4 2024-04-02T15:24:29.223 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: mon.a@-1(???).mds e1 new map 2024-04-02T15:24:29.223 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: mon.a@-1(???).mds e1 print_map 2024-04-02T15:24:29.224 INFO:journalctl@ceph.mon.a.smithi087.stdout: e1 2024-04-02T15:24:29.224 INFO:journalctl@ceph.mon.a.smithi087.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-02T15:24:29.224 INFO:journalctl@ceph.mon.a.smithi087.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-02T15:24:29.224 INFO:journalctl@ceph.mon.a.smithi087.stdout: legacy client fscid: -1 2024-04-02T15:24:29.224 INFO:journalctl@ceph.mon.a.smithi087.stdout: 2024-04-02T15:24:29.224 INFO:journalctl@ceph.mon.a.smithi087.stdout: No filesystems configured 2024-04-02T15:24:29.224 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-02T15:24:29.224 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-02T15:24:29.224 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-02T15:24:29.224 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-02T15:24:29.224 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-04-02T15:24:29.224 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: expand_channel_meta expand map: {default=false} 2024-04-02T15:24:29.224 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: expand_channel_meta from 'false' to 'false' 2024-04-02T15:24:29.224 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: expand_channel_meta expanded map: {default=false} 2024-04-02T15:24:29.225 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: expand_channel_meta expand map: {default=info} 2024-04-02T15:24:29.225 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: expand_channel_meta from 'info' to 'info' 2024-04-02T15:24:29.225 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: expand_channel_meta expanded map: {default=info} 2024-04-02T15:24:29.225 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: expand_channel_meta expand map: {default=daemon} 2024-04-02T15:24:29.225 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: expand_channel_meta from 'daemon' to 'daemon' 2024-04-02T15:24:29.225 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: expand_channel_meta expanded map: {default=daemon} 2024-04-02T15:24:29.225 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: expand_channel_meta expand map: {default=debug} 2024-04-02T15:24:29.225 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: expand_channel_meta from 'debug' to 'debug' 2024-04-02T15:24:29.225 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: expand_channel_meta expanded map: {default=debug} 2024-04-02T15:24:29.225 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-02T15:24:29.225 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: monmap e1: 1 mons at {a=[v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0]} 2024-04-02T15:24:29.225 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: fsmap 2024-04-02T15:24:29.225 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: osdmap e1: 0 total, 0 up, 0 in 2024-04-02T15:24:29.226 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 ceph-mon[24900]: mgrmap e1: no daemons active 2024-04-02T15:24:29.226 INFO:teuthology.orchestra.run.smithi087.stderr:Setting mon public_network to 172.21.0.0/20 2024-04-02T15:24:29.457 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 bash[24879]: e28fd32a7d9a16e9cd29d93249526e5ab8e7c89a1154253f4f465df687030731 2024-04-02T15:24:29.457 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:29 smithi087 systemd[1]: Started Ceph mon.a for 0400842c-f105-11ee-b647-cb9ed24678a4. 2024-04-02T15:24:30.229 INFO:teuthology.orchestra.run.smithi087.stderr:Wrote config to /etc/ceph/ceph.conf 2024-04-02T15:24:30.229 INFO:teuthology.orchestra.run.smithi087.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-04-02T15:24:30.229 INFO:teuthology.orchestra.run.smithi087.stderr:Creating mgr... 2024-04-02T15:24:30.230 INFO:teuthology.orchestra.run.smithi087.stderr:Verifying port 9283 ... 2024-04-02T15:24:30.401 INFO:teuthology.orchestra.run.smithi087.stderr:systemctl: Failed to reset failed state of unit ceph-0400842c-f105-11ee-b647-cb9ed24678a4@mgr.y.service: Unit ceph-0400842c-f105-11ee-b647-cb9ed24678a4@mgr.y.service not loaded. 2024-04-02T15:24:30.408 INFO:teuthology.orchestra.run.smithi087.stderr:systemctl: Created symlink /etc/systemd/system/ceph-0400842c-f105-11ee-b647-cb9ed24678a4.target.wants/ceph-0400842c-f105-11ee-b647-cb9ed24678a4@mgr.y.service → /etc/systemd/system/ceph-0400842c-f105-11ee-b647-cb9ed24678a4@.service. 2024-04-02T15:24:30.869 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:30 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/3326195481' entity='client.admin' 2024-04-02T15:24:30.869 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:30 smithi087 podman[25118]: 2024-04-02 15:24:30.800048497 +0000 UTC m=+0.135710546 container create 5d422ff3cc7d951a9ea798f2aa50cf7abb9d711f0fdd5d165830165c8dc0ed88 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y, io.k8s.display-name=CentOS Stream 8, name=centos-stream, vcs-type=git, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, release=754) 2024-04-02T15:24:31.127 INFO:teuthology.orchestra.run.smithi087.stderr:systemctl: disabled 2024-04-02T15:24:31.134 INFO:teuthology.orchestra.run.smithi087.stderr:systemctl: inactive 2024-04-02T15:24:31.134 INFO:teuthology.orchestra.run.smithi087.stderr:firewalld.service is not enabled 2024-04-02T15:24:31.134 INFO:teuthology.orchestra.run.smithi087.stderr:Not possible to enable service . firewalld.service is not available 2024-04-02T15:24:31.142 INFO:teuthology.orchestra.run.smithi087.stderr:systemctl: disabled 2024-04-02T15:24:31.149 INFO:teuthology.orchestra.run.smithi087.stderr:systemctl: inactive 2024-04-02T15:24:31.150 INFO:teuthology.orchestra.run.smithi087.stderr:firewalld.service is not enabled 2024-04-02T15:24:31.150 INFO:teuthology.orchestra.run.smithi087.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-04-02T15:24:31.150 INFO:teuthology.orchestra.run.smithi087.stderr:Waiting for mgr to start... 2024-04-02T15:24:31.150 INFO:teuthology.orchestra.run.smithi087.stderr:Waiting for mgr... 2024-04-02T15:24:31.256 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:31 smithi087 podman[25118]: 2024-04-02 15:24:31.019163951 +0000 UTC m=+0.354826001 container init 5d422ff3cc7d951a9ea798f2aa50cf7abb9d711f0fdd5d165830165c8dc0ed88 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, ceph=True, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, RELEASE=HEAD, release=754, vcs-type=git, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8) 2024-04-02T15:24:31.256 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:31 smithi087 podman[25118]: 2024-04-02 15:24:31.021677458 +0000 UTC m=+0.357339517 container start 5d422ff3cc7d951a9ea798f2aa50cf7abb9d711f0fdd5d165830165c8dc0ed88 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, vcs-type=git, ceph=True, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc.) 2024-04-02T15:24:31.256 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:31 smithi087 bash[25118]: 5d422ff3cc7d951a9ea798f2aa50cf7abb9d711f0fdd5d165830165c8dc0ed88 2024-04-02T15:24:31.256 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:31 smithi087 systemd[1]: Started Ceph mgr.y for 0400842c-f105-11ee-b647-cb9ed24678a4. 2024-04-02T15:24:31.639 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: 2024-04-02T15:24:31.639 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: { 2024-04-02T15:24:31.639 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "fsid": "0400842c-f105-11ee-b647-cb9ed24678a4", 2024-04-02T15:24:31.639 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "health": { 2024-04-02T15:24:31.639 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-02T15:24:31.639 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "checks": {}, 2024-04-02T15:24:31.639 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "mutes": [] 2024-04-02T15:24:31.640 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: }, 2024-04-02T15:24:31.640 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-02T15:24:31.640 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "quorum": [ 2024-04-02T15:24:31.640 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: 0 2024-04-02T15:24:31.640 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: ], 2024-04-02T15:24:31.640 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-02T15:24:31.640 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "a" 2024-04-02T15:24:31.640 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: ], 2024-04-02T15:24:31.640 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-04-02T15:24:31.640 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "monmap": { 2024-04-02T15:24:31.640 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T15:24:31.640 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-02T15:24:31.640 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-02T15:24:31.640 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: }, 2024-04-02T15:24:31.640 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "osdmap": { 2024-04-02T15:24:31.641 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T15:24:31.641 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-02T15:24:31.641 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-02T15:24:31.641 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-02T15:24:31.641 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-02T15:24:31.641 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-02T15:24:31.641 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-02T15:24:31.641 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: }, 2024-04-02T15:24:31.641 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "pgmap": { 2024-04-02T15:24:31.641 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-02T15:24:31.641 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-02T15:24:31.641 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-02T15:24:31.641 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-02T15:24:31.641 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-02T15:24:31.641 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-02T15:24:31.642 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-02T15:24:31.642 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-02T15:24:31.642 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: }, 2024-04-02T15:24:31.642 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "fsmap": { 2024-04-02T15:24:31.642 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T15:24:31.642 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-02T15:24:31.642 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-02T15:24:31.642 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: }, 2024-04-02T15:24:31.642 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-02T15:24:31.642 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "available": false, 2024-04-02T15:24:31.642 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-02T15:24:31.642 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "modules": [ 2024-04-02T15:24:31.642 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "iostat", 2024-04-02T15:24:31.643 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "nfs", 2024-04-02T15:24:31.643 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "restful" 2024-04-02T15:24:31.643 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: ], 2024-04-02T15:24:31.643 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "services": {} 2024-04-02T15:24:31.643 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: }, 2024-04-02T15:24:31.643 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "servicemap": { 2024-04-02T15:24:31.643 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T15:24:31.643 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "modified": "2024-04-02T15:24:24.701599+0000", 2024-04-02T15:24:31.643 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "services": {} 2024-04-02T15:24:31.643 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: }, 2024-04-02T15:24:31.643 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-02T15:24:31.643 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: } 2024-04-02T15:24:31.868 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:31 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/3383560748' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-02T15:24:31.868 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:31 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:31.510+0000 7fa0c2bd2000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-02T15:24:31.869 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:31 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:31.602+0000 7fa0c2bd2000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-02T15:24:32.180 INFO:teuthology.orchestra.run.smithi087.stderr:mgr not available, waiting (1/15)... 2024-04-02T15:24:32.524 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:32 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:32.209+0000 7fa0c2bd2000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-02T15:24:32.524 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:32 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:32.298+0000 7fa0c2bd2000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-02T15:24:32.524 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:32 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:32.381+0000 7fa0c2bd2000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-02T15:24:32.792 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:32 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:32.528+0000 7fa0c2bd2000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-02T15:24:32.792 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:32 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:32.608+0000 7fa0c2bd2000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-02T15:24:32.792 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:32 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:32.715+0000 7fa0c2bd2000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-02T15:24:32.792 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:32 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:32.796+0000 7fa0c2bd2000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-02T15:24:33.118 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:32 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:32.956+0000 7fa0c2bd2000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-02T15:24:33.118 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:33 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:33.039+0000 7fa0c2bd2000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-02T15:24:34.368 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:34 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:34.073+0000 7fa0c2bd2000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-02T15:24:34.368 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:34 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:34.210+0000 7fa0c2bd2000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-02T15:24:34.868 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:34 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:34.404+0000 7fa0c2bd2000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-02T15:24:34.868 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:34 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:34.489+0000 7fa0c2bd2000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-02T15:24:34.869 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:34 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:34.621+0000 7fa0c2bd2000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-02T15:24:35.228 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:34 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:34.934+0000 7fa0c2bd2000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-02T15:24:35.228 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:35 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:35.149+0000 7fa0c2bd2000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-02T15:24:35.228 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:35 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:35.232+0000 7fa0c2bd2000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-02T15:24:35.618 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:35 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:35.401+0000 7fa0c2bd2000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-02T15:24:36.597 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:36 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:36.308+0000 7fa0c2bd2000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-02T15:24:36.597 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:36 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:36.601+0000 7fa0c2bd2000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-02T15:24:36.868 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:36 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:36.695+0000 7fa0c2bd2000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-02T15:24:36.868 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:36 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:36.844+0000 7fa0c2bd2000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-02T15:24:37.368 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:36 smithi087 ceph-mon[24900]: Activating manager daemon y 2024-04-02T15:24:37.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:36 smithi087 ceph-mon[24900]: mgrmap e2: y(active, starting, since 0.00273307s) 2024-04-02T15:24:37.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:36 smithi087 ceph-mon[24900]: from='mgr.14100 172.21.15.87:0/2696097042' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T15:24:37.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:36 smithi087 ceph-mon[24900]: from='mgr.14100 172.21.15.87:0/2696097042' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T15:24:37.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:36 smithi087 ceph-mon[24900]: from='mgr.14100 172.21.15.87:0/2696097042' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T15:24:37.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:36 smithi087 ceph-mon[24900]: from='mgr.14100 172.21.15.87:0/2696097042' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T15:24:37.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:36 smithi087 ceph-mon[24900]: from='mgr.14100 172.21.15.87:0/2696097042' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T15:24:37.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:36 smithi087 ceph-mon[24900]: Manager daemon y is now available 2024-04-02T15:24:37.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:36 smithi087 ceph-mon[24900]: from='mgr.14100 172.21.15.87:0/2696097042' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:24:37.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:36 smithi087 ceph-mon[24900]: from='mgr.14100 172.21.15.87:0/2696097042' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:24:37.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:36 smithi087 ceph-mon[24900]: from='mgr.14100 172.21.15.87:0/2696097042' entity='mgr.y' 2024-04-02T15:24:37.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:36 smithi087 ceph-mon[24900]: from='mgr.14100 172.21.15.87:0/2696097042' entity='mgr.y' 2024-04-02T15:24:37.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:36 smithi087 ceph-mon[24900]: from='mgr.14100 172.21.15.87:0/2696097042' entity='mgr.y' 2024-04-02T15:24:38.368 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:37 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/1245194282' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-02T15:24:38.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:37 smithi087 ceph-mon[24900]: mgrmap e3: y(active, since 1.00607s) 2024-04-02T15:24:38.529 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: 2024-04-02T15:24:38.530 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: { 2024-04-02T15:24:38.530 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "fsid": "0400842c-f105-11ee-b647-cb9ed24678a4", 2024-04-02T15:24:38.530 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "health": { 2024-04-02T15:24:38.530 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-02T15:24:38.530 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "checks": {}, 2024-04-02T15:24:38.530 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "mutes": [] 2024-04-02T15:24:38.530 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: }, 2024-04-02T15:24:38.530 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-02T15:24:38.530 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "quorum": [ 2024-04-02T15:24:38.530 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: 0 2024-04-02T15:24:38.530 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: ], 2024-04-02T15:24:38.530 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-02T15:24:38.530 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "a" 2024-04-02T15:24:38.530 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: ], 2024-04-02T15:24:38.531 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "quorum_age": 8, 2024-04-02T15:24:38.531 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "monmap": { 2024-04-02T15:24:38.531 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T15:24:38.531 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-02T15:24:38.531 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-02T15:24:38.531 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: }, 2024-04-02T15:24:38.531 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "osdmap": { 2024-04-02T15:24:38.531 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T15:24:38.531 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-02T15:24:38.531 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-02T15:24:38.531 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-02T15:24:38.531 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-02T15:24:38.534 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-02T15:24:38.534 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-02T15:24:38.535 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: }, 2024-04-02T15:24:38.535 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "pgmap": { 2024-04-02T15:24:38.535 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-02T15:24:38.535 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-02T15:24:38.535 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-02T15:24:38.535 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-02T15:24:38.535 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-02T15:24:38.535 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-02T15:24:38.535 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-02T15:24:38.535 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-02T15:24:38.535 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: }, 2024-04-02T15:24:38.535 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "fsmap": { 2024-04-02T15:24:38.535 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T15:24:38.535 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-02T15:24:38.535 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-02T15:24:38.535 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: }, 2024-04-02T15:24:38.536 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-02T15:24:38.536 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "available": false, 2024-04-02T15:24:38.536 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-02T15:24:38.536 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "modules": [ 2024-04-02T15:24:38.536 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "iostat", 2024-04-02T15:24:38.536 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "nfs", 2024-04-02T15:24:38.536 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "restful" 2024-04-02T15:24:38.536 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: ], 2024-04-02T15:24:38.536 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "services": {} 2024-04-02T15:24:38.536 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: }, 2024-04-02T15:24:38.536 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "servicemap": { 2024-04-02T15:24:38.536 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T15:24:38.536 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "modified": "2024-04-02T15:24:24.701599+0000", 2024-04-02T15:24:38.536 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "services": {} 2024-04-02T15:24:38.536 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: }, 2024-04-02T15:24:38.536 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-02T15:24:38.536 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: } 2024-04-02T15:24:40.368 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:39 smithi087 ceph-mon[24900]: mgrmap e4: y(active, since 2s) 2024-04-02T15:24:43.227 INFO:teuthology.orchestra.run.smithi087.stderr:mgr not available, waiting (2/15)... 2024-04-02T15:24:45.860 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: 2024-04-02T15:24:45.860 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: { 2024-04-02T15:24:45.860 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "fsid": "0400842c-f105-11ee-b647-cb9ed24678a4", 2024-04-02T15:24:45.860 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "health": { 2024-04-02T15:24:45.860 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-02T15:24:45.860 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "checks": {}, 2024-04-02T15:24:45.860 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "mutes": [] 2024-04-02T15:24:45.860 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: }, 2024-04-02T15:24:45.860 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-02T15:24:45.860 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "quorum": [ 2024-04-02T15:24:45.860 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: 0 2024-04-02T15:24:45.860 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: ], 2024-04-02T15:24:45.860 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-02T15:24:45.861 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "a" 2024-04-02T15:24:45.861 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: ], 2024-04-02T15:24:45.861 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "quorum_age": 16, 2024-04-02T15:24:45.861 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "monmap": { 2024-04-02T15:24:45.861 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T15:24:45.861 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-02T15:24:45.861 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-02T15:24:45.861 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: }, 2024-04-02T15:24:45.861 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "osdmap": { 2024-04-02T15:24:45.861 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T15:24:45.861 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-02T15:24:45.861 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-02T15:24:45.861 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-02T15:24:45.861 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-02T15:24:45.865 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-02T15:24:45.865 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-02T15:24:45.865 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: }, 2024-04-02T15:24:45.865 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "pgmap": { 2024-04-02T15:24:45.865 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-02T15:24:45.865 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-02T15:24:45.865 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-02T15:24:45.865 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-02T15:24:45.865 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-02T15:24:45.865 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-02T15:24:45.865 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-02T15:24:45.866 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-02T15:24:45.866 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: }, 2024-04-02T15:24:45.866 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "fsmap": { 2024-04-02T15:24:45.866 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T15:24:45.866 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-02T15:24:45.866 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-02T15:24:45.866 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: }, 2024-04-02T15:24:45.866 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-02T15:24:45.866 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "available": true, 2024-04-02T15:24:45.866 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-02T15:24:45.866 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "modules": [ 2024-04-02T15:24:45.866 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "iostat", 2024-04-02T15:24:45.866 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "nfs", 2024-04-02T15:24:45.866 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "restful" 2024-04-02T15:24:45.866 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: ], 2024-04-02T15:24:45.867 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "services": {} 2024-04-02T15:24:45.867 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: }, 2024-04-02T15:24:45.867 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "servicemap": { 2024-04-02T15:24:45.867 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-02T15:24:45.867 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "modified": "2024-04-02T15:24:24.701599+0000", 2024-04-02T15:24:45.867 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "services": {} 2024-04-02T15:24:45.867 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: }, 2024-04-02T15:24:45.867 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-02T15:24:45.867 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: } 2024-04-02T15:24:46.118 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:45 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/3775555070' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-02T15:24:46.406 INFO:teuthology.orchestra.run.smithi087.stderr:mgr is available 2024-04-02T15:24:46.993 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: 2024-04-02T15:24:46.993 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: [global] 2024-04-02T15:24:46.993 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: fsid = 0400842c-f105-11ee-b647-cb9ed24678a4 2024-04-02T15:24:46.993 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-04-02T15:24:46.993 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-04-02T15:24:46.993 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-04-02T15:24:46.993 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-04-02T15:24:46.993 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: 2024-04-02T15:24:46.993 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: [mgr] 2024-04-02T15:24:46.993 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-04-02T15:24:46.993 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: 2024-04-02T15:24:46.994 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: [osd] 2024-04-02T15:24:46.994 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-04-02T15:24:46.994 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-04-02T15:24:47.352 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:47 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/1186059204' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-02T15:24:47.352 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:47 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/1186059204' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-02T15:24:47.512 INFO:teuthology.orchestra.run.smithi087.stderr:Enabling cephadm module... 2024-04-02T15:24:48.618 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:48 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/2188966441' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-04-02T15:24:48.619 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:48 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ignoring --setuser ceph since I am not root 2024-04-02T15:24:48.619 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:48 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ignoring --setgroup ceph since I am not root 2024-04-02T15:24:49.118 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:48 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:48.709+0000 7f3729c0f000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-02T15:24:49.119 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:48 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:48.800+0000 7f3729c0f000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-02T15:24:49.473 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:49 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:49.389+0000 7f3729c0f000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-02T15:24:49.474 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:49 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/2188966441' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-04-02T15:24:49.474 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:49 smithi087 ceph-mon[24900]: mgrmap e5: y(active, since 11s) 2024-04-02T15:24:49.750 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:49 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:49.478+0000 7f3729c0f000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-02T15:24:49.750 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:49 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:49.561+0000 7f3729c0f000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-02T15:24:49.750 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:49 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:49.710+0000 7f3729c0f000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-02T15:24:50.118 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:49 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:49.791+0000 7f3729c0f000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-02T15:24:50.119 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:49 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:49.899+0000 7f3729c0f000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-02T15:24:50.119 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:49 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:49.983+0000 7f3729c0f000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-02T15:24:50.423 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:50 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:50.147+0000 7f3729c0f000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-02T15:24:50.423 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:50 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:50.233+0000 7f3729c0f000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-02T15:24:51.119 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: { 2024-04-02T15:24:51.119 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "epoch": 5, 2024-04-02T15:24:51.119 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "available": true, 2024-04-02T15:24:51.120 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "active_name": "y", 2024-04-02T15:24:51.120 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-02T15:24:51.120 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: } 2024-04-02T15:24:51.412 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:51 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/1502847252' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-02T15:24:51.413 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:51 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:51.281+0000 7f3729c0f000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-02T15:24:51.413 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:51 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:51.417+0000 7f3729c0f000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-02T15:24:51.632 INFO:teuthology.orchestra.run.smithi087.stderr:Waiting for the mgr to restart... 2024-04-02T15:24:51.632 INFO:teuthology.orchestra.run.smithi087.stderr:Waiting for mgr epoch 5... 2024-04-02T15:24:51.702 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:51 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:51.608+0000 7f3729c0f000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-02T15:24:51.702 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:51 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:51.707+0000 7f3729c0f000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-02T15:24:52.023 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:51 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:51.844+0000 7f3729c0f000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-02T15:24:52.368 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:52 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:52.165+0000 7f3729c0f000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-02T15:24:52.634 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:52 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:52.383+0000 7f3729c0f000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-02T15:24:52.634 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:52 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:52.468+0000 7f3729c0f000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-02T15:24:52.634 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:52 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:52.638+0000 7f3729c0f000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-02T15:24:53.834 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:53 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:53.548+0000 7f3729c0f000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-02T15:24:53.834 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:53 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:53.837+0000 7f3729c0f000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-02T15:24:54.118 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:53 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:53.931+0000 7f3729c0f000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-02T15:24:54.119 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:54 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:24:54.080+0000 7f3729c0f000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-02T15:24:55.140 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: { 2024-04-02T15:24:55.140 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-04-02T15:24:55.140 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "initialized": true 2024-04-02T15:24:55.140 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: } 2024-04-02T15:24:55.369 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:55 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: [02/Apr/2024:15:24:55] ENGINE Bus STARTING 2024-04-02T15:24:55.369 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:55 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: [02/Apr/2024:15:24:55] ENGINE Serving on https://172.21.15.87:7150 2024-04-02T15:24:55.369 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:55 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: [02/Apr/2024:15:24:55] ENGINE Bus STARTED 2024-04-02T15:24:55.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:55 smithi087 ceph-mon[24900]: Active manager daemon y restarted 2024-04-02T15:24:55.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:55 smithi087 ceph-mon[24900]: Activating manager daemon y 2024-04-02T15:24:55.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:55 smithi087 ceph-mon[24900]: osdmap e2: 0 total, 0 up, 0 in 2024-04-02T15:24:55.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:55 smithi087 ceph-mon[24900]: mgrmap e6: y(active, starting, since 0.0524876s) 2024-04-02T15:24:55.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:55 smithi087 ceph-mon[24900]: from='mgr.14118 172.21.15.87:0/2846236556' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T15:24:55.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:55 smithi087 ceph-mon[24900]: from='mgr.14118 172.21.15.87:0/2846236556' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T15:24:55.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:55 smithi087 ceph-mon[24900]: from='mgr.14118 172.21.15.87:0/2846236556' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T15:24:55.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:55 smithi087 ceph-mon[24900]: from='mgr.14118 172.21.15.87:0/2846236556' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T15:24:55.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:55 smithi087 ceph-mon[24900]: from='mgr.14118 172.21.15.87:0/2846236556' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T15:24:55.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:55 smithi087 ceph-mon[24900]: Manager daemon y is now available 2024-04-02T15:24:55.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:55 smithi087 ceph-mon[24900]: from='mgr.14118 172.21.15.87:0/2846236556' entity='mgr.y' 2024-04-02T15:24:55.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:55 smithi087 ceph-mon[24900]: from='mgr.14118 172.21.15.87:0/2846236556' entity='mgr.y' 2024-04-02T15:24:55.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:55 smithi087 ceph-mon[24900]: from='mgr.14118 172.21.15.87:0/2846236556' entity='mgr.y' 2024-04-02T15:24:55.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:55 smithi087 ceph-mon[24900]: from='mgr.14118 172.21.15.87:0/2846236556' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:24:55.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:55 smithi087 ceph-mon[24900]: from='mgr.14118 172.21.15.87:0/2846236556' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:24:55.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:55 smithi087 ceph-mon[24900]: from='mgr.14118 172.21.15.87:0/2846236556' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:24:55.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:55 smithi087 ceph-mon[24900]: from='mgr.14118 172.21.15.87:0/2846236556' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:24:55.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:55 smithi087 ceph-mon[24900]: from='mgr.14118 172.21.15.87:0/2846236556' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:24:55.720 INFO:teuthology.orchestra.run.smithi087.stderr:mgr epoch 5 is available 2024-04-02T15:24:55.720 INFO:teuthology.orchestra.run.smithi087.stderr:Setting orchestrator backend to cephadm... 2024-04-02T15:24:56.321 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:56 smithi087 ceph-mon[24900]: [02/Apr/2024:15:24:55] ENGINE Bus STARTING 2024-04-02T15:24:56.321 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:56 smithi087 ceph-mon[24900]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-02T15:24:56.321 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:56 smithi087 ceph-mon[24900]: mgrmap e7: y(active, since 1.05489s) 2024-04-02T15:24:56.321 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:56 smithi087 ceph-mon[24900]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-02T15:24:56.321 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:56 smithi087 ceph-mon[24900]: [02/Apr/2024:15:24:55] ENGINE Serving on https://172.21.15.87:7150 2024-04-02T15:24:56.321 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:56 smithi087 ceph-mon[24900]: [02/Apr/2024:15:24:55] ENGINE Bus STARTED 2024-04-02T15:24:56.321 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:56 smithi087 ceph-mon[24900]: from='mgr.14118 172.21.15.87:0/2846236556' entity='mgr.y' 2024-04-02T15:24:56.321 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:56 smithi087 ceph-mon[24900]: from='mgr.14118 172.21.15.87:0/2846236556' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:24:57.443 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:57 smithi087 ceph-mon[24900]: mgrmap e8: y(active, since 2s) 2024-04-02T15:24:57.443 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:57 smithi087 ceph-mon[24900]: from='mgr.14118 172.21.15.87:0/2846236556' entity='mgr.y' 2024-04-02T15:24:57.443 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:57 smithi087 ceph-mon[24900]: from='mgr.14118 172.21.15.87:0/2846236556' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:24:57.443 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: value unchanged 2024-04-02T15:24:58.016 INFO:teuthology.orchestra.run.smithi087.stderr:Generating ssh key... 2024-04-02T15:24:58.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:58 smithi087 ceph-mon[24900]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:24:59.618 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:24:59 smithi087 ceph-mon[24900]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:24:59.619 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:59 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: Generating public/private rsa key pair. 2024-04-02T15:24:59.619 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:59 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: Your identification has been saved in /tmp/tmpp_eoero8/key. 2024-04-02T15:24:59.619 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:59 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: Your public key has been saved in /tmp/tmpp_eoero8/key.pub. 2024-04-02T15:24:59.619 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:59 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: The key fingerprint is: 2024-04-02T15:24:59.619 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:59 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: SHA256:OzjAppiRRVL3Z252fe3xpIqnvT/fk0CKpAYmFrKvm3Y ceph-0400842c-f105-11ee-b647-cb9ed24678a4 2024-04-02T15:24:59.619 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:59 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: The key's randomart image is: 2024-04-02T15:24:59.619 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:59 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: +---[RSA 3072]----+ 2024-04-02T15:24:59.619 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:59 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: |..o . | 2024-04-02T15:24:59.619 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:59 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: | + o . | 2024-04-02T15:24:59.619 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:59 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: | + . . o | 2024-04-02T15:24:59.620 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:59 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: | + + o + . . . . | 2024-04-02T15:24:59.620 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:59 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: |o o * . S o + ..o| 2024-04-02T15:24:59.620 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:59 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: | + + . * + . o +o| 2024-04-02T15:24:59.620 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:59 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: |o o + o o +| 2024-04-02T15:24:59.620 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:59 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: | o.E . . o...o.| 2024-04-02T15:24:59.620 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:59 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: |.oo oo+o.o+| 2024-04-02T15:24:59.620 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:24:59 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: +----[SHA256]-----+ 2024-04-02T15:25:00.520 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:00 smithi087 ceph-mon[24900]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:25:00.521 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:00 smithi087 ceph-mon[24900]: Generating ssh key... 2024-04-02T15:25:00.521 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:00 smithi087 ceph-mon[24900]: from='mgr.14118 172.21.15.87:0/2846236556' entity='mgr.y' 2024-04-02T15:25:00.521 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:00 smithi087 ceph-mon[24900]: from='mgr.14118 172.21.15.87:0/2846236556' entity='mgr.y' 2024-04-02T15:25:00.521 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC+QCshBI7E+gSzhS6Vw01lDuZDzUalZpwTN944JOKoPt8WfPzm5YPFf/YguFqHN4onWskOjF9/HM6Mw6wroELPofgCrtp2YmDeZHhL8DOb0pon4hd3T3wtpksyaG705mIu5rUwpSl94ToM08QvqZJw+32cQcAf7HVHwzfTZFZrycTSfhtgJuXR+jMUsOx7CPt2dPoR4vjv/QCGiSciBbyYpk86s8V27kmH7X1+JayGsXXzJBdiokCQS0Bv1ZiPGKZ5nJdaH48PHhn5eHKln0T247nKWFmm0Jl0czQnPBmz/SrxjGwuINuIR62eFza/Dfj38UGm4UQSeNdW+Un0/JJFKXoWaJiZefpDLh8huk8AfHWCXoqRu7Ugr3rFX2EDhr2HtAtLbDf0e//l9yXRyY0kGw/02Af9a+tSfMbnmx99cQpYcDP24F+9v4DA86lfKMBPNvCFYCN1kikmQjmVuTz8K58G8TXxt62FNjsjRAnp1EiF4lJTHm4yBZ2NNPYX/2E= ceph-0400842c-f105-11ee-b647-cb9ed24678a4 2024-04-02T15:25:01.061 INFO:teuthology.orchestra.run.smithi087.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-04-02T15:25:01.061 INFO:teuthology.orchestra.run.smithi087.stderr:Adding key to root@localhost authorized_keys... 2024-04-02T15:25:01.061 INFO:teuthology.orchestra.run.smithi087.stderr:Adding host smithi087... 2024-04-02T15:25:01.368 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:01 smithi087 ceph-mon[24900]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:25:02.618 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:02 smithi087 ceph-mon[24900]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi087", "addr": "172.21.15.87", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:25:02.619 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:02 smithi087 ceph-mon[24900]: Deploying cephadm binary to smithi087 2024-04-02T15:25:02.683 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: Added host 'smithi087' with addr '172.21.15.87' 2024-04-02T15:25:03.284 INFO:teuthology.orchestra.run.smithi087.stderr:Deploying unmanaged mon service... 2024-04-02T15:25:03.918 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:03 smithi087 ceph-mon[24900]: from='mgr.14118 172.21.15.87:0/2846236556' entity='mgr.y' 2024-04-02T15:25:03.918 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:03 smithi087 ceph-mon[24900]: Added host smithi087 2024-04-02T15:25:03.918 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:03 smithi087 ceph-mon[24900]: from='mgr.14118 172.21.15.87:0/2846236556' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:25:03.919 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: Scheduled mon update... 2024-04-02T15:25:04.633 INFO:teuthology.orchestra.run.smithi087.stderr:Deploying unmanaged mgr service... 2024-04-02T15:25:05.219 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:04 smithi087 ceph-mon[24900]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:25:05.219 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:04 smithi087 ceph-mon[24900]: Saving service mon spec with placement count:5 2024-04-02T15:25:05.219 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:04 smithi087 ceph-mon[24900]: from='mgr.14118 172.21.15.87:0/2846236556' entity='mgr.y' 2024-04-02T15:25:05.230 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-04-02T15:25:06.510 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:06 smithi087 ceph-mon[24900]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:25:06.510 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:06 smithi087 ceph-mon[24900]: Saving service mgr spec with placement count:2 2024-04-02T15:25:06.510 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:06 smithi087 ceph-mon[24900]: from='mgr.14118 172.21.15.87:0/2846236556' entity='mgr.y' 2024-04-02T15:25:07.868 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:07 smithi087 ceph-mon[24900]: from='mgr.14118 172.21.15.87:0/2846236556' entity='mgr.y' 2024-04-02T15:25:07.869 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:07 smithi087 ceph-mon[24900]: from='mgr.14118 172.21.15.87:0/2846236556' entity='mgr.y' 2024-04-02T15:25:07.869 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:07 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/2439926417' entity='client.admin' 2024-04-02T15:25:08.855 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:08 smithi087 ceph-mon[24900]: from='mgr.14118 172.21.15.87:0/2846236556' entity='mgr.y' 2024-04-02T15:25:08.855 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:08 smithi087 ceph-mon[24900]: from='mgr.14118 172.21.15.87:0/2846236556' entity='mgr.y' 2024-04-02T15:25:08.855 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:08 smithi087 ceph-mon[24900]: from='mgr.14118 172.21.15.87:0/2846236556' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi087", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:25:08.855 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:08 smithi087 ceph-mon[24900]: from='mgr.14118 172.21.15.87:0/2846236556' entity='mgr.y' 2024-04-02T15:25:08.855 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:08 smithi087 ceph-mon[24900]: from='mgr.14118 172.21.15.87:0/2846236556' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi087", "caps": []}]: dispatch 2024-04-02T15:25:08.855 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:08 smithi087 ceph-mon[24900]: from='mgr.14118 172.21.15.87:0/2846236556' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.agent.smithi087", "caps": []}]': finished 2024-04-02T15:25:08.856 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:08 smithi087 ceph-mon[24900]: Deploying daemon agent.smithi087 on smithi087 2024-04-02T15:25:09.603 INFO:teuthology.orchestra.run.smithi087.stderr:Enabling the dashboard module... 2024-04-02T15:25:09.852 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:09 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/1443440855' entity='client.admin' 2024-04-02T15:25:11.118 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:10 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/2672040353' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-04-02T15:25:11.119 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:25:10 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ignoring --setuser ceph since I am not root 2024-04-02T15:25:11.119 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:25:10 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ignoring --setgroup ceph since I am not root 2024-04-02T15:25:11.438 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:25:11 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:25:11.353+0000 7fd9539e8000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-02T15:25:11.704 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:25:11 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:25:11.443+0000 7fd9539e8000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-02T15:25:12.059 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:11 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/2672040353' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-04-02T15:25:12.059 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:11 smithi087 ceph-mon[24900]: mgrmap e9: y(active, since 16s) 2024-04-02T15:25:12.155 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: { 2024-04-02T15:25:12.155 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "epoch": 9, 2024-04-02T15:25:12.155 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "available": true, 2024-04-02T15:25:12.155 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "active_name": "y", 2024-04-02T15:25:12.155 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-02T15:25:12.155 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: } 2024-04-02T15:25:12.369 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:25:12 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:25:12.063+0000 7fd9539e8000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-02T15:25:12.369 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:25:12 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:25:12.154+0000 7fd9539e8000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-02T15:25:12.369 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:25:12 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:25:12.241+0000 7fd9539e8000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-02T15:25:12.662 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:25:12 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:25:12.392+0000 7fd9539e8000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-02T15:25:12.662 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:25:12 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:25:12.474+0000 7fd9539e8000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-02T15:25:12.662 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:25:12 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:25:12.583+0000 7fd9539e8000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-02T15:25:12.662 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:25:12 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:25:12.667+0000 7fd9539e8000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-02T15:25:12.733 INFO:teuthology.orchestra.run.smithi087.stderr:Waiting for the mgr to restart... 2024-04-02T15:25:12.733 INFO:teuthology.orchestra.run.smithi087.stderr:Waiting for mgr epoch 9... 2024-04-02T15:25:12.917 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:12 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/3623565248' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-02T15:25:12.917 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:25:12 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:25:12.834+0000 7fd9539e8000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-02T15:25:13.369 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:25:12 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:25:12.921+0000 7fd9539e8000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-02T15:25:14.267 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:25:13 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:25:13.952+0000 7fd9539e8000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-02T15:25:14.267 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:25:14 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:25:14.082+0000 7fd9539e8000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-02T15:25:14.268 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:25:14 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:25:14.272+0000 7fd9539e8000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-02T15:25:14.619 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:25:14 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:25:14.358+0000 7fd9539e8000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-02T15:25:14.619 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:25:14 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:25:14.492+0000 7fd9539e8000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-02T15:25:15.111 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:25:14 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:25:14.810+0000 7fd9539e8000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-02T15:25:15.111 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:25:15 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:25:15.030+0000 7fd9539e8000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-02T15:25:15.111 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:25:15 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:25:15.115+0000 7fd9539e8000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-02T15:25:15.368 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:25:15 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:25:15.293+0000 7fd9539e8000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-02T15:25:16.539 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:25:16 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:25:16.241+0000 7fd9539e8000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-02T15:25:16.838 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:25:16 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:25:16.540+0000 7fd9539e8000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-02T15:25:16.839 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:25:16 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:25:16.635+0000 7fd9539e8000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-02T15:25:16.839 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:25:16 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:25:16.783+0000 7fd9539e8000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-02T15:25:17.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:16 smithi087 ceph-mon[24900]: Active manager daemon y restarted 2024-04-02T15:25:17.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:16 smithi087 ceph-mon[24900]: Activating manager daemon y 2024-04-02T15:25:17.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:16 smithi087 ceph-mon[24900]: osdmap e3: 0 total, 0 up, 0 in 2024-04-02T15:25:17.842 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:25:17 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: [02/Apr/2024:15:25:17] ENGINE Bus STARTING 2024-04-02T15:25:17.842 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:25:17 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: [02/Apr/2024:15:25:17] ENGINE Serving on https://172.21.15.87:7150 2024-04-02T15:25:17.842 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:25:17 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: [02/Apr/2024:15:25:17] ENGINE Bus STARTED 2024-04-02T15:25:17.856 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: { 2024-04-02T15:25:17.856 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-04-02T15:25:17.856 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: "initialized": true 2024-04-02T15:25:17.856 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: } 2024-04-02T15:25:18.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:17 smithi087 ceph-mon[24900]: mgrmap e10: y(active, starting, since 0.0556291s) 2024-04-02T15:25:18.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:17 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T15:25:18.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:17 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T15:25:18.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:17 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T15:25:18.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:17 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T15:25:18.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:17 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T15:25:18.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:17 smithi087 ceph-mon[24900]: Manager daemon y is now available 2024-04-02T15:25:18.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:17 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:18.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:17 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:25:18.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:17 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:25:18.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:17 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:25:18.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:17 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:25:18.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:17 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:18.500 INFO:teuthology.orchestra.run.smithi087.stderr:mgr epoch 9 is available 2024-04-02T15:25:18.500 INFO:teuthology.orchestra.run.smithi087.stderr:Generating a dashboard self-signed certificate... 2024-04-02T15:25:19.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:18 smithi087 ceph-mon[24900]: [02/Apr/2024:15:25:17] ENGINE Bus STARTING 2024-04-02T15:25:19.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:18 smithi087 ceph-mon[24900]: [02/Apr/2024:15:25:17] ENGINE Serving on https://172.21.15.87:7150 2024-04-02T15:25:19.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:18 smithi087 ceph-mon[24900]: [02/Apr/2024:15:25:17] ENGINE Bus STARTED 2024-04-02T15:25:19.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:18 smithi087 ceph-mon[24900]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-02T15:25:19.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:18 smithi087 ceph-mon[24900]: mgrmap e11: y(active, since 1.06908s) 2024-04-02T15:25:19.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:18 smithi087 ceph-mon[24900]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-02T15:25:19.191 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: Self-signed certificate created 2024-04-02T15:25:19.773 INFO:teuthology.orchestra.run.smithi087.stderr:Creating initial admin user... 2024-04-02T15:25:20.073 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:19 smithi087 ceph-mon[24900]: mgrmap e12: y(active, since 2s) 2024-04-02T15:25:20.074 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:19 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:20.074 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:19 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi087", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:25:20.074 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:19 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:20.074 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:19 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi087", "caps": []}]: dispatch 2024-04-02T15:25:20.074 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:19 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:20.074 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:19 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:20.582 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$IrnNzN6tD4pqM4l5xdM8a.1Uhr6n52aGzL/V7BuueVS9J5Dm73zSi", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1712071520, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-04-02T15:25:21.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:20 smithi087 ceph-mon[24900]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:25:21.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:20 smithi087 ceph-mon[24900]: Deploying daemon agent.smithi087 on smithi087 2024-04-02T15:25:21.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:20 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:21.188 INFO:teuthology.orchestra.run.smithi087.stderr:Fetching dashboard port number... 2024-04-02T15:25:21.868 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: 8443 2024-04-02T15:25:22.118 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:21 smithi087 ceph-mon[24900]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:25:22.461 INFO:teuthology.orchestra.run.smithi087.stderr:systemctl: disabled 2024-04-02T15:25:22.468 INFO:teuthology.orchestra.run.smithi087.stderr:systemctl: inactive 2024-04-02T15:25:22.468 INFO:teuthology.orchestra.run.smithi087.stderr:firewalld.service is not enabled 2024-04-02T15:25:22.468 INFO:teuthology.orchestra.run.smithi087.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-04-02T15:25:22.470 INFO:teuthology.orchestra.run.smithi087.stderr:Ceph Dashboard is now available at: 2024-04-02T15:25:22.470 INFO:teuthology.orchestra.run.smithi087.stderr: 2024-04-02T15:25:22.470 INFO:teuthology.orchestra.run.smithi087.stderr: URL: https://smithi087.front.sepia.ceph.com:8443/ 2024-04-02T15:25:22.470 INFO:teuthology.orchestra.run.smithi087.stderr: User: admin 2024-04-02T15:25:22.470 INFO:teuthology.orchestra.run.smithi087.stderr: Password: ur9ebg2wkq 2024-04-02T15:25:22.471 INFO:teuthology.orchestra.run.smithi087.stderr: 2024-04-02T15:25:22.471 INFO:teuthology.orchestra.run.smithi087.stderr:Enabling autotune for osd_memory_target 2024-04-02T15:25:23.106 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:22 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/3715777008' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-04-02T15:25:23.996 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:23 smithi087 ceph-mon[24900]: mgrmap e13: y(active, since 6s) 2024-04-02T15:25:24.502 INFO:teuthology.orchestra.run.smithi087.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-04-02T15:25:25.158 INFO:teuthology.orchestra.run.smithi087.stderr:You can access the Ceph CLI with: 2024-04-02T15:25:25.158 INFO:teuthology.orchestra.run.smithi087.stderr: 2024-04-02T15:25:25.158 INFO:teuthology.orchestra.run.smithi087.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-04-02T15:25:25.158 INFO:teuthology.orchestra.run.smithi087.stderr: 2024-04-02T15:25:25.158 INFO:teuthology.orchestra.run.smithi087.stderr:Please consider enabling telemetry to help improve Ceph: 2024-04-02T15:25:25.158 INFO:teuthology.orchestra.run.smithi087.stderr: 2024-04-02T15:25:25.158 INFO:teuthology.orchestra.run.smithi087.stderr: ceph telemetry on 2024-04-02T15:25:25.158 INFO:teuthology.orchestra.run.smithi087.stderr: 2024-04-02T15:25:25.158 INFO:teuthology.orchestra.run.smithi087.stderr:For more information see: 2024-04-02T15:25:25.158 INFO:teuthology.orchestra.run.smithi087.stderr: 2024-04-02T15:25:25.158 INFO:teuthology.orchestra.run.smithi087.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-04-02T15:25:25.158 INFO:teuthology.orchestra.run.smithi087.stderr: 2024-04-02T15:25:25.158 INFO:teuthology.orchestra.run.smithi087.stderr:Bootstrap complete. 2024-04-02T15:25:25.193 INFO:tasks.cephadm:Fetching config... 2024-04-02T15:25:25.193 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2024-04-02T15:25:25.193 DEBUG:teuthology.orchestra.run.smithi087:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-04-02T15:25:25.208 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-04-02T15:25:25.208 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2024-04-02T15:25:25.208 DEBUG:teuthology.orchestra.run.smithi087:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-04-02T15:25:25.268 INFO:tasks.cephadm:Fetching mon keyring... 2024-04-02T15:25:25.268 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2024-04-02T15:25:25.268 DEBUG:teuthology.orchestra.run.smithi087:> sudo dd if=/var/lib/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/mon.a/keyring of=/dev/stdout 2024-04-02T15:25:25.375 INFO:tasks.cephadm:Fetching pub ssh key... 2024-04-02T15:25:25.375 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2024-04-02T15:25:25.375 DEBUG:teuthology.orchestra.run.smithi087:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-04-02T15:25:25.430 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-04-02T15:25:25.517 DEBUG:teuthology.orchestra.run.smithi087:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC+QCshBI7E+gSzhS6Vw01lDuZDzUalZpwTN944JOKoPt8WfPzm5YPFf/YguFqHN4onWskOjF9/HM6Mw6wroELPofgCrtp2YmDeZHhL8DOb0pon4hd3T3wtpksyaG705mIu5rUwpSl94ToM08QvqZJw+32cQcAf7HVHwzfTZFZrycTSfhtgJuXR+jMUsOx7CPt2dPoR4vjv/QCGiSciBbyYpk86s8V27kmH7X1+JayGsXXzJBdiokCQS0Bv1ZiPGKZ5nJdaH48PHhn5eHKln0T247nKWFmm0Jl0czQnPBmz/SrxjGwuINuIR62eFza/Dfj38UGm4UQSeNdW+Un0/JJFKXoWaJiZefpDLh8huk8AfHWCXoqRu7Ugr3rFX2EDhr2HtAtLbDf0e//l9yXRyY0kGw/02Af9a+tSfMbnmx99cQpYcDP24F+9v4DA86lfKMBPNvCFYCN1kikmQjmVuTz8K58G8TXxt62FNjsjRAnp1EiF4lJTHm4yBZ2NNPYX/2E= ceph-0400842c-f105-11ee-b647-cb9ed24678a4' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-02T15:25:25.593 INFO:teuthology.orchestra.run.smithi087.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC+QCshBI7E+gSzhS6Vw01lDuZDzUalZpwTN944JOKoPt8WfPzm5YPFf/YguFqHN4onWskOjF9/HM6Mw6wroELPofgCrtp2YmDeZHhL8DOb0pon4hd3T3wtpksyaG705mIu5rUwpSl94ToM08QvqZJw+32cQcAf7HVHwzfTZFZrycTSfhtgJuXR+jMUsOx7CPt2dPoR4vjv/QCGiSciBbyYpk86s8V27kmH7X1+JayGsXXzJBdiokCQS0Bv1ZiPGKZ5nJdaH48PHhn5eHKln0T247nKWFmm0Jl0czQnPBmz/SrxjGwuINuIR62eFza/Dfj38UGm4UQSeNdW+Un0/JJFKXoWaJiZefpDLh8huk8AfHWCXoqRu7Ugr3rFX2EDhr2HtAtLbDf0e//l9yXRyY0kGw/02Af9a+tSfMbnmx99cQpYcDP24F+9v4DA86lfKMBPNvCFYCN1kikmQjmVuTz8K58G8TXxt62FNjsjRAnp1EiF4lJTHm4yBZ2NNPYX/2E= ceph-0400842c-f105-11ee-b647-cb9ed24678a4 2024-04-02T15:25:25.608 DEBUG:teuthology.orchestra.run.smithi106:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC+QCshBI7E+gSzhS6Vw01lDuZDzUalZpwTN944JOKoPt8WfPzm5YPFf/YguFqHN4onWskOjF9/HM6Mw6wroELPofgCrtp2YmDeZHhL8DOb0pon4hd3T3wtpksyaG705mIu5rUwpSl94ToM08QvqZJw+32cQcAf7HVHwzfTZFZrycTSfhtgJuXR+jMUsOx7CPt2dPoR4vjv/QCGiSciBbyYpk86s8V27kmH7X1+JayGsXXzJBdiokCQS0Bv1ZiPGKZ5nJdaH48PHhn5eHKln0T247nKWFmm0Jl0czQnPBmz/SrxjGwuINuIR62eFza/Dfj38UGm4UQSeNdW+Un0/JJFKXoWaJiZefpDLh8huk8AfHWCXoqRu7Ugr3rFX2EDhr2HtAtLbDf0e//l9yXRyY0kGw/02Af9a+tSfMbnmx99cQpYcDP24F+9v4DA86lfKMBPNvCFYCN1kikmQjmVuTz8K58G8TXxt62FNjsjRAnp1EiF4lJTHm4yBZ2NNPYX/2E= ceph-0400842c-f105-11ee-b647-cb9ed24678a4' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-02T15:25:25.656 INFO:teuthology.orchestra.run.smithi106.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC+QCshBI7E+gSzhS6Vw01lDuZDzUalZpwTN944JOKoPt8WfPzm5YPFf/YguFqHN4onWskOjF9/HM6Mw6wroELPofgCrtp2YmDeZHhL8DOb0pon4hd3T3wtpksyaG705mIu5rUwpSl94ToM08QvqZJw+32cQcAf7HVHwzfTZFZrycTSfhtgJuXR+jMUsOx7CPt2dPoR4vjv/QCGiSciBbyYpk86s8V27kmH7X1+JayGsXXzJBdiokCQS0Bv1ZiPGKZ5nJdaH48PHhn5eHKln0T247nKWFmm0Jl0czQnPBmz/SrxjGwuINuIR62eFza/Dfj38UGm4UQSeNdW+Un0/JJFKXoWaJiZefpDLh8huk8AfHWCXoqRu7Ugr3rFX2EDhr2HtAtLbDf0e//l9yXRyY0kGw/02Af9a+tSfMbnmx99cQpYcDP24F+9v4DA86lfKMBPNvCFYCN1kikmQjmVuTz8K58G8TXxt62FNjsjRAnp1EiF4lJTHm4yBZ2NNPYX/2E= ceph-0400842c-f105-11ee-b647-cb9ed24678a4 2024-04-02T15:25:25.672 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-04-02T15:25:25.753 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:25 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/4207495348' entity='client.admin' 2024-04-02T15:25:26.619 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:25:26 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:25:26] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:25:27.332 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:27 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:27.332 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:27 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:25:27.332 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:27 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:27.332 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:27 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:25:27.332 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:27 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:27.332 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:27 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:27.332 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:27 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:30.205 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:30 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/3467700594' entity='client.admin' 2024-04-02T15:25:30.205 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:30 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:25:30.205 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:30 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:30.483 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:25:30 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:25:30] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:25:30.508 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-04-02T15:25:30.508 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-04-02T15:25:31.094 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:25:30 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:25:30] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:25:31.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:31 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:31.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:31 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:31.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:31 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:33.027 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi106 2024-04-02T15:25:33.027 DEBUG:teuthology.orchestra.run.smithi106:> set -ex 2024-04-02T15:25:33.027 DEBUG:teuthology.orchestra.run.smithi106:> dd of=/etc/ceph/ceph.conf 2024-04-02T15:25:33.045 DEBUG:teuthology.orchestra.run.smithi106:> set -ex 2024-04-02T15:25:33.045 DEBUG:teuthology.orchestra.run.smithi106:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-02T15:25:33.103 INFO:tasks.cephadm:Adding host smithi106 to orchestrator... 2024-04-02T15:25:33.103 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph orch host add smithi106 2024-04-02T15:25:33.869 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:33 smithi087 ceph-mon[24900]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:25:33.869 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:33 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:33.869 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:33 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:25:33.869 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:33 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:25:33.869 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:33 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:25:33.869 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:33 smithi087 ceph-mon[24900]: Updating smithi087:/etc/ceph/ceph.conf 2024-04-02T15:25:33.869 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:33 smithi087 ceph-mon[24900]: Updating smithi087:/etc/ceph/ceph.client.admin.keyring 2024-04-02T15:25:33.869 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:33 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:33.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:33 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:35.619 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:35 smithi087 ceph-mon[24900]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi106", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:25:35.799 INFO:teuthology.orchestra.run.smithi087.stdout:Added host 'smithi106' with addr '172.21.15.106' 2024-04-02T15:25:36.399 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph orch host ls --format=json 2024-04-02T15:25:37.027 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:36 smithi087 ceph-mon[24900]: Deploying cephadm binary to smithi106 2024-04-02T15:25:37.027 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:36 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:37.027 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:36 smithi087 ceph-mon[24900]: Added host smithi106 2024-04-02T15:25:37.027 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:36 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:25:37.027 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:36 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:25:37.027 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:36 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:25:37.028 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:36 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:37.028 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:36 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:37.028 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:36 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:37.028 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:36 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi106", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:25:37.028 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:36 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:37.028 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:36 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi106", "caps": []}]: dispatch 2024-04-02T15:25:37.028 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:36 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.agent.smithi106", "caps": []}]': finished 2024-04-02T15:25:37.994 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:37 smithi087 ceph-mon[24900]: Deploying daemon agent.smithi106 on smithi106 2024-04-02T15:25:37.994 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:37 smithi087 ceph-mon[24900]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:25:37.994 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:37 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:37.994 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:37 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:25:37.994 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:37 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:25:37.994 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:37 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:25:38.277 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-02T15:25:38.277 INFO:teuthology.orchestra.run.smithi087.stdout:[{"addr": "172.21.15.87", "hostname": "smithi087", "labels": [], "status": ""}, {"addr": "172.21.15.106", "hostname": "smithi106", "labels": [], "status": ""}] 2024-04-02T15:25:38.277 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:25:38 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:25:38] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:25:38.889 INFO:tasks.cephadm:Setting crush tunables to default 2024-04-02T15:25:38.889 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph osd crush tunables default 2024-04-02T15:25:39.119 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:25:38 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:25:38] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:25:39.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:38 smithi087 ceph-mon[24900]: Updating smithi106:/etc/ceph/ceph.conf 2024-04-02T15:25:39.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:38 smithi087 ceph-mon[24900]: Updating smithi106:/etc/ceph/ceph.client.admin.keyring 2024-04-02T15:25:39.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:38 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:39.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:38 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:39.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:38 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:25:39.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:38 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:25:39.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:38 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:25:39.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:38 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:39.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:38 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:40.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:39 smithi087 ceph-mon[24900]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-02T15:25:40.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:39 smithi087 ceph-mon[24900]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:25:40.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:39 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:40.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:39 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:40.507 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:25:40 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:25:40] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:25:41.337 INFO:teuthology.orchestra.run.smithi087.stderr:adjusted tunables profile to default 2024-04-02T15:25:41.619 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:41 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:41.619 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:41 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:41.619 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:41 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/2590555919' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-02T15:25:41.619 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:41 smithi087 ceph-mon[24900]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:25:42.023 INFO:tasks.cephadm:Adding mon.a on smithi087 2024-04-02T15:25:42.023 INFO:tasks.cephadm:Adding mon.c on smithi087 2024-04-02T15:25:42.023 INFO:tasks.cephadm:Adding mon.b on smithi106 2024-04-02T15:25:42.023 DEBUG:teuthology.orchestra.run.smithi106:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph orch apply mon '3;smithi087:172.21.15.87=a;smithi087:[v2:172.21.15.87:3301,v1:172.21.15.87:6790]=c;smithi106:172.21.15.106=b' 2024-04-02T15:25:42.619 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:42 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/2590555919' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-02T15:25:42.619 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:42 smithi087 ceph-mon[24900]: osdmap e4: 0 total, 0 up, 0 in 2024-04-02T15:25:42.619 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:42 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:43.619 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:43 smithi087 ceph-mon[24900]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:25:43.739 INFO:teuthology.orchestra.run.smithi106.stdout:Scheduled mon update... 2024-04-02T15:25:44.314 DEBUG:teuthology.orchestra.run.smithi087:mon.c> sudo journalctl -f -n 0 -u ceph-0400842c-f105-11ee-b647-cb9ed24678a4@mon.c.service 2024-04-02T15:25:44.316 DEBUG:teuthology.orchestra.run.smithi106:mon.b> sudo journalctl -f -n 0 -u ceph-0400842c-f105-11ee-b647-cb9ed24678a4@mon.b.service 2024-04-02T15:25:44.319 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-02T15:25:44.319 DEBUG:teuthology.orchestra.run.smithi106:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-04-02T15:25:44.737 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:44 smithi087 ceph-mon[24900]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi087:172.21.15.87=a;smithi087:[v2:172.21.15.87:3301,v1:172.21.15.87:6790]=c;smithi106:172.21.15.106=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:25:44.738 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:44 smithi087 ceph-mon[24900]: Saving service mon spec with placement smithi087:172.21.15.87=a;smithi087:[v2:172.21.15.87:3301,v1:172.21.15.87:6790]=c;smithi106:172.21.15.106=b;count:3 2024-04-02T15:25:44.738 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:44 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:44.738 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:44 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:25:44.738 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:44 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:25:44.738 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:44 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:25:44.738 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:44 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:44.738 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:44 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T15:25:44.738 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:44 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:25:44.738 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:44 smithi087 ceph-mon[24900]: Deploying daemon mon.c on smithi087 2024-04-02T15:25:45.991 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:45 smithi087 ceph-mon[24900]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:25:46.105 INFO:teuthology.orchestra.run.smithi106.stdout: 2024-04-02T15:25:46.105 INFO:teuthology.orchestra.run.smithi106.stdout:{"epoch":1,"fsid":"0400842c-f105-11ee-b647-cb9ed24678a4","modified":"2024-04-02T15:24:21.862790Z","created":"2024-04-02T15:24:21.862790Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:3300","nonce":0},{"type":"v1","addr":"172.21.15.87:6789","nonce":0}]},"addr":"172.21.15.87:6789/0","public_addr":"172.21.15.87:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-02T15:25:46.106 INFO:teuthology.orchestra.run.smithi106.stderr:dumped monmap epoch 1 2024-04-02T15:25:46.834 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:46 smithi087 ceph-mon[24900]: from='client.? 172.21.15.106:0/1266727001' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T15:25:46.835 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:25:46 smithi087 ceph-mon[29707]: mon.c@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-04-02T15:25:47.119 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:25:46 smithi087 bash[29666]: 40f23934618c40010db6cb10ca7763742ac86a7b63337e73ae3f5c1fb728ea81 2024-04-02T15:25:47.119 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:25:46 smithi087 systemd[1]: Started Ceph mon.c for 0400842c-f105-11ee-b647-cb9ed24678a4. 2024-04-02T15:25:47.661 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-02T15:25:47.661 DEBUG:teuthology.orchestra.run.smithi106:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-04-02T15:25:52.119 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:25:51] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:25:52.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T15:25:52.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[24900]: mon.a calling monitor election 2024-04-02T15:25:52.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:25:52.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[24900]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:25:52.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:25:52.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:25:52.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[24900]: mon.c calling monitor election 2024-04-02T15:25:52.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[24900]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:25:52.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:25:52.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:25:52.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[24900]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:25:52.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:25:52.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[24900]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-02T15:25:52.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[24900]: monmap e2: 2 mons at {a=[v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0],c=[v2:172.21.15.87:3301/0,v1:172.21.15.87:6790/0]} 2024-04-02T15:25:52.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[24900]: fsmap 2024-04-02T15:25:52.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[24900]: osdmap e4: 0 total, 0 up, 0 in 2024-04-02T15:25:52.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[24900]: mgrmap e13: y(active, since 35s) 2024-04-02T15:25:52.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[24900]: overall HEALTH_OK 2024-04-02T15:25:52.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:52.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:52.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T15:25:52.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:25:52.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:52.122 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T15:25:52.122 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[29707]: mon.a calling monitor election 2024-04-02T15:25:52.122 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:25:52.122 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[29707]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:25:52.122 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:25:52.122 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:25:52.122 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[29707]: mon.c calling monitor election 2024-04-02T15:25:52.122 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[29707]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:25:52.122 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:25:52.122 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:25:52.122 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[29707]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:25:52.122 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:25:52.122 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[29707]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-02T15:25:52.122 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[29707]: monmap e2: 2 mons at {a=[v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0],c=[v2:172.21.15.87:3301/0,v1:172.21.15.87:6790/0]} 2024-04-02T15:25:52.123 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[29707]: fsmap 2024-04-02T15:25:52.123 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[29707]: osdmap e4: 0 total, 0 up, 0 in 2024-04-02T15:25:52.123 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[29707]: mgrmap e13: y(active, since 35s) 2024-04-02T15:25:52.123 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[29707]: overall HEALTH_OK 2024-04-02T15:25:52.123 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:52.123 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:52.123 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T15:25:52.123 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:25:52.123 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:25:51 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:52.547 INFO:teuthology.orchestra.run.smithi106.stdout: 2024-04-02T15:25:52.547 INFO:teuthology.orchestra.run.smithi106.stdout:{"epoch":2,"fsid":"0400842c-f105-11ee-b647-cb9ed24678a4","modified":"2024-04-02T15:25:46.804376Z","created":"2024-04-02T15:24:21.862790Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:3300","nonce":0},{"type":"v1","addr":"172.21.15.87:6789","nonce":0}]},"addr":"172.21.15.87:6789/0","public_addr":"172.21.15.87:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:3301","nonce":0},{"type":"v1","addr":"172.21.15.87:6790","nonce":0}]},"addr":"172.21.15.87:6790/0","public_addr":"172.21.15.87:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-02T15:25:52.549 INFO:teuthology.orchestra.run.smithi106.stderr:dumped monmap epoch 2 2024-04-02T15:25:53.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:52 smithi087 ceph-mon[24900]: Deploying daemon mon.b on smithi106 2024-04-02T15:25:53.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:52 smithi087 ceph-mon[24900]: from='client.? 172.21.15.106:0/1665489746' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T15:25:53.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:52 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:25:53.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:25:52 smithi087 ceph-mon[29707]: Deploying daemon mon.b on smithi106 2024-04-02T15:25:53.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:25:52 smithi087 ceph-mon[29707]: from='client.? 172.21.15.106:0/1665489746' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T15:25:53.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:25:52 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:25:54.207 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-02T15:25:54.207 DEBUG:teuthology.orchestra.run.smithi106:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-04-02T15:25:54.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:53 smithi087 ceph-mon[24900]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:25:54.369 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:25:53 smithi087 ceph-mon[29707]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:25:55.619 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:25:55 smithi087 ceph-mon[24900]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:25:55.619 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:25:55 smithi087 ceph-mon[29707]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:25:55.672 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:55 smithi106 systemd[1]: Starting Ceph mon.b for 0400842c-f105-11ee-b647-cb9ed24678a4... 2024-04-02T15:25:56.581 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 podman[25803]: 2024-04-02 15:25:56.399808612 +0000 UTC m=+0.142230056 container create d9de5ac6a73e6f45899b23f50d1cac0fe7f5d0af2f64640f7348af83fe702801 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mon-b, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, version=8, release=754, maintainer=Guillaume Abrioux , name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, vendor=Red Hat, Inc.) 2024-04-02T15:25:56.599 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 podman[25803]: 2024-04-02 15:25:56.329418978 +0000 UTC m=+0.071840443 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-02T15:25:56.599 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 podman[25803]: 2024-04-02 15:25:56.585611688 +0000 UTC m=+0.328033138 container init d9de5ac6a73e6f45899b23f50d1cac0fe7f5d0af2f64640f7348af83fe702801 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mon-b, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, name=centos-stream, io.k8s.display-name=CentOS Stream 8, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.openshift.expose-services=, RELEASE=HEAD, ceph=True, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container) 2024-04-02T15:25:56.599 INFO:teuthology.orchestra.run.smithi106.stdout: 2024-04-02T15:25:56.600 INFO:teuthology.orchestra.run.smithi106.stdout:{"epoch":2,"fsid":"0400842c-f105-11ee-b647-cb9ed24678a4","modified":"2024-04-02T15:25:46.804376Z","created":"2024-04-02T15:24:21.862790Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:3300","nonce":0},{"type":"v1","addr":"172.21.15.87:6789","nonce":0}]},"addr":"172.21.15.87:6789/0","public_addr":"172.21.15.87:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:3301","nonce":0},{"type":"v1","addr":"172.21.15.87:6790","nonce":0}]},"addr":"172.21.15.87:6790/0","public_addr":"172.21.15.87:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-02T15:25:56.602 INFO:teuthology.orchestra.run.smithi106.stderr:dumped monmap epoch 2 2024-04-02T15:25:56.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 podman[25803]: 2024-04-02 15:25:56.588763707 +0000 UTC m=+0.331185151 container start d9de5ac6a73e6f45899b23f50d1cac0fe7f5d0af2f64640f7348af83fe702801 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mon-b, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, ceph=True, version=8, GIT_BRANCH=HEAD, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git) 2024-04-02T15:25:56.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: set uid:gid to 167:167 (ceph:ceph) 2024-04-02T15:25:56.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-04-02T15:25:56.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: pidfile_write: ignore empty --pid-file 2024-04-02T15:25:56.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: load: jerasure load: lrc 2024-04-02T15:25:56.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: RocksDB version: 6.15.5 2024-04-02T15:25:56.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-02T15:25:56.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Compile date Apr 18 2022 2024-04-02T15:25:56.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: DB SUMMARY 2024-04-02T15:25:56.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: DB Session ID: QF6OQV7X15MF3GC5A1KE 2024-04-02T15:25:56.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: CURRENT file: CURRENT 2024-04-02T15:25:56.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: IDENTITY file: IDENTITY 2024-04-02T15:25:56.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-04-02T15:25:56.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2024-04-02T15:25:56.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 636 ; 2024-04-02T15:25:56.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.error_if_exists: 0 2024-04-02T15:25:56.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.create_if_missing: 0 2024-04-02T15:25:56.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.paranoid_checks: 1 2024-04-02T15:25:56.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-02T15:25:56.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.env: 0x560d644c6860 2024-04-02T15:25:56.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.fs: Posix File System 2024-04-02T15:25:56.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.info_log: 0x560d6506ddc0 2024-04-02T15:25:56.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.max_file_opening_threads: 16 2024-04-02T15:25:56.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.statistics: (nil) 2024-04-02T15:25:56.868 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.use_fsync: 0 2024-04-02T15:25:56.868 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.max_log_file_size: 0 2024-04-02T15:25:56.868 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-02T15:25:56.868 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-02T15:25:56.868 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.keep_log_file_num: 1000 2024-04-02T15:25:56.868 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.recycle_log_file_num: 0 2024-04-02T15:25:56.868 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.allow_fallocate: 1 2024-04-02T15:25:56.868 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.allow_mmap_reads: 0 2024-04-02T15:25:56.868 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.allow_mmap_writes: 0 2024-04-02T15:25:56.868 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.use_direct_reads: 0 2024-04-02T15:25:56.868 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-02T15:25:56.868 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.create_missing_column_families: 0 2024-04-02T15:25:56.868 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.db_log_dir: 2024-04-02T15:25:56.868 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2024-04-02T15:25:56.868 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-02T15:25:56.868 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-02T15:25:56.869 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-02T15:25:56.869 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-02T15:25:56.869 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-02T15:25:56.869 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-02T15:25:56.869 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.advise_random_on_open: 1 2024-04-02T15:25:56.869 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.db_write_buffer_size: 0 2024-04-02T15:25:56.869 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.write_buffer_manager: 0x560d6515e240 2024-04-02T15:25:56.869 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-02T15:25:56.869 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-02T15:25:56.869 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-02T15:25:56.869 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-02T15:25:56.869 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.rate_limiter: (nil) 2024-04-02T15:25:56.869 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-02T15:25:56.869 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.wal_recovery_mode: 2 2024-04-02T15:25:56.869 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.enable_thread_tracking: 0 2024-04-02T15:25:56.870 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.enable_pipelined_write: 0 2024-04-02T15:25:56.870 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.unordered_write: 0 2024-04-02T15:25:56.870 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-02T15:25:56.870 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-02T15:25:56.870 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-02T15:25:56.870 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-02T15:25:56.870 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.row_cache: None 2024-04-02T15:25:56.871 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.wal_filter: None 2024-04-02T15:25:56.871 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-02T15:25:56.871 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.allow_ingest_behind: 0 2024-04-02T15:25:56.872 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.preserve_deletes: 0 2024-04-02T15:25:56.872 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.two_write_queues: 0 2024-04-02T15:25:56.872 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.manual_wal_flush: 0 2024-04-02T15:25:56.872 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.atomic_flush: 0 2024-04-02T15:25:56.872 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-02T15:25:56.872 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-02T15:25:56.872 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-02T15:25:56.872 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.log_readahead_size: 0 2024-04-02T15:25:56.872 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-02T15:25:56.872 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.best_efforts_recovery: 0 2024-04-02T15:25:56.872 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-02T15:25:56.872 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-02T15:25:56.873 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.allow_data_in_errors: 0 2024-04-02T15:25:56.873 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.db_host_id: __hostname__ 2024-04-02T15:25:56.873 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.max_background_jobs: 2 2024-04-02T15:25:56.873 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.max_background_compactions: -1 2024-04-02T15:25:56.873 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.max_subcompactions: 1 2024-04-02T15:25:56.873 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-02T15:25:56.873 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-02T15:25:56.873 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-02T15:25:56.873 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.max_total_wal_size: 0 2024-04-02T15:25:56.873 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-02T15:25:56.873 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-02T15:25:56.873 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-02T15:25:56.873 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-02T15:25:56.873 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.max_open_files: -1 2024-04-02T15:25:56.873 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.bytes_per_sync: 0 2024-04-02T15:25:56.874 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-02T15:25:56.874 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-02T15:25:56.874 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.compaction_readahead_size: 0 2024-04-02T15:25:56.874 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.max_background_flushes: -1 2024-04-02T15:25:56.874 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Compression algorithms supported: 2024-04-02T15:25:56.874 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-02T15:25:56.874 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: kZSTD supported: 0 2024-04-02T15:25:56.874 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: kXpressCompression supported: 0 2024-04-02T15:25:56.874 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: kLZ4HCCompression supported: 1 2024-04-02T15:25:56.874 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: kLZ4Compression supported: 1 2024-04-02T15:25:56.874 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: kBZip2Compression supported: 0 2024-04-02T15:25:56.874 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: kZlibCompression supported: 1 2024-04-02T15:25:56.874 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: kSnappyCompression supported: 1 2024-04-02T15:25:56.874 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-02T15:25:56.874 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2024-04-02T15:25:56.875 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-04-02T15:25:56.875 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-02T15:25:56.875 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.merge_operator: 2024-04-02T15:25:56.875 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.compaction_filter: None 2024-04-02T15:25:56.875 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.compaction_filter_factory: None 2024-04-02T15:25:56.875 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.sst_partitioner_factory: None 2024-04-02T15:25:56.875 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-02T15:25:56.875 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-02T15:25:56.875 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x560d65039d00) 2024-04-02T15:25:56.875 INFO:journalctl@ceph.mon.b.smithi106.stdout: cache_index_and_filter_blocks: 1 2024-04-02T15:25:56.875 INFO:journalctl@ceph.mon.b.smithi106.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-02T15:25:56.876 INFO:journalctl@ceph.mon.b.smithi106.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-02T15:25:56.876 INFO:journalctl@ceph.mon.b.smithi106.stdout: pin_top_level_index_and_filter: 1 2024-04-02T15:25:56.876 INFO:journalctl@ceph.mon.b.smithi106.stdout: index_type: 0 2024-04-02T15:25:56.876 INFO:journalctl@ceph.mon.b.smithi106.stdout: data_block_index_type: 0 2024-04-02T15:25:56.876 INFO:journalctl@ceph.mon.b.smithi106.stdout: index_shortening: 1 2024-04-02T15:25:56.876 INFO:journalctl@ceph.mon.b.smithi106.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-02T15:25:56.876 INFO:journalctl@ceph.mon.b.smithi106.stdout: hash_index_allow_collision: 1 2024-04-02T15:25:56.876 INFO:journalctl@ceph.mon.b.smithi106.stdout: checksum: 1 2024-04-02T15:25:56.876 INFO:journalctl@ceph.mon.b.smithi106.stdout: no_block_cache: 0 2024-04-02T15:25:56.876 INFO:journalctl@ceph.mon.b.smithi106.stdout: block_cache: 0x560d650a4170 2024-04-02T15:25:56.876 INFO:journalctl@ceph.mon.b.smithi106.stdout: block_cache_name: BinnedLRUCache 2024-04-02T15:25:56.876 INFO:journalctl@ceph.mon.b.smithi106.stdout: block_cache_options: 2024-04-02T15:25:56.876 INFO:journalctl@ceph.mon.b.smithi106.stdout: capacity : 536870912 2024-04-02T15:25:56.876 INFO:journalctl@ceph.mon.b.smithi106.stdout: num_shard_bits : 4 2024-04-02T15:25:56.876 INFO:journalctl@ceph.mon.b.smithi106.stdout: strict_capacity_limit : 0 2024-04-02T15:25:56.877 INFO:journalctl@ceph.mon.b.smithi106.stdout: high_pri_pool_ratio: 0.000 2024-04-02T15:25:56.877 INFO:journalctl@ceph.mon.b.smithi106.stdout: block_cache_compressed: (nil) 2024-04-02T15:25:56.877 INFO:journalctl@ceph.mon.b.smithi106.stdout: persistent_cache: (nil) 2024-04-02T15:25:56.877 INFO:journalctl@ceph.mon.b.smithi106.stdout: block_size: 4096 2024-04-02T15:25:56.877 INFO:journalctl@ceph.mon.b.smithi106.stdout: block_size_deviation: 10 2024-04-02T15:25:56.877 INFO:journalctl@ceph.mon.b.smithi106.stdout: block_restart_interval: 16 2024-04-02T15:25:56.877 INFO:journalctl@ceph.mon.b.smithi106.stdout: index_block_restart_interval: 1 2024-04-02T15:25:56.877 INFO:journalctl@ceph.mon.b.smithi106.stdout: metadata_block_size: 4096 2024-04-02T15:25:56.877 INFO:journalctl@ceph.mon.b.smithi106.stdout: partition_filters: 0 2024-04-02T15:25:56.877 INFO:journalctl@ceph.mon.b.smithi106.stdout: use_delta_encoding: 1 2024-04-02T15:25:56.877 INFO:journalctl@ceph.mon.b.smithi106.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-02T15:25:56.877 INFO:journalctl@ceph.mon.b.smithi106.stdout: whole_key_filtering: 1 2024-04-02T15:25:56.877 INFO:journalctl@ceph.mon.b.smithi106.stdout: verify_compression: 0 2024-04-02T15:25:56.877 INFO:journalctl@ceph.mon.b.smithi106.stdout: read_amp_bytes_per_bit: 0 2024-04-02T15:25:56.878 INFO:journalctl@ceph.mon.b.smithi106.stdout: format_version: 4 2024-04-02T15:25:56.878 INFO:journalctl@ceph.mon.b.smithi106.stdout: enable_index_compression: 1 2024-04-02T15:25:56.878 INFO:journalctl@ceph.mon.b.smithi106.stdout: block_align: 0 2024-04-02T15:25:56.878 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.write_buffer_size: 33554432 2024-04-02T15:25:56.878 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.max_write_buffer_number: 2 2024-04-02T15:25:56.878 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.compression: NoCompression 2024-04-02T15:25:56.878 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.bottommost_compression: Disabled 2024-04-02T15:25:56.878 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.prefix_extractor: nullptr 2024-04-02T15:25:56.878 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-02T15:25:56.878 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.num_levels: 7 2024-04-02T15:25:56.878 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-02T15:25:56.878 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-02T15:25:56.878 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-02T15:25:56.878 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-02T15:25:56.879 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-02T15:25:56.879 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-02T15:25:56.879 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-02T15:25:56.879 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-02T15:25:56.879 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-02T15:25:56.879 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-02T15:25:56.879 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-02T15:25:56.879 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.compression_opts.level: 32767 2024-04-02T15:25:56.879 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.compression_opts.strategy: 0 2024-04-02T15:25:56.879 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-02T15:25:56.879 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-02T15:25:56.879 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-02T15:25:56.879 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.compression_opts.enabled: false 2024-04-02T15:25:56.879 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-02T15:25:56.879 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-02T15:25:56.880 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-02T15:25:56.880 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.target_file_size_base: 67108864 2024-04-02T15:25:56.880 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-02T15:25:56.880 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-02T15:25:56.880 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-02T15:25:56.880 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-02T15:25:56.880 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-02T15:25:56.880 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-02T15:25:56.880 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-02T15:25:56.880 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-02T15:25:56.880 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-02T15:25:56.880 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-02T15:25:56.880 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-02T15:25:56.880 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-02T15:25:56.880 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-02T15:25:56.880 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.arena_block_size: 4194304 2024-04-02T15:25:56.881 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-02T15:25:56.881 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-02T15:25:56.881 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-02T15:25:56.881 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.disable_auto_compactions: 0 2024-04-02T15:25:56.881 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-02T15:25:56.881 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-02T15:25:56.881 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-02T15:25:56.881 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-02T15:25:56.881 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-02T15:25:56.881 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-02T15:25:56.881 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-02T15:25:56.881 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-02T15:25:56.881 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-02T15:25:56.881 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-02T15:25:56.881 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.table_properties_collectors: 2024-04-02T15:25:56.882 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.inplace_update_support: 0 2024-04-02T15:25:56.882 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-02T15:25:56.882 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-02T15:25:56.882 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-02T15:25:56.882 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-02T15:25:56.882 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.bloom_locality: 0 2024-04-02T15:25:56.882 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.max_successive_merges: 0 2024-04-02T15:25:56.882 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-02T15:25:56.882 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.paranoid_file_checks: 0 2024-04-02T15:25:56.882 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.force_consistency_checks: 1 2024-04-02T15:25:56.882 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.report_bg_io_stats: 0 2024-04-02T15:25:56.882 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.ttl: 2592000 2024-04-02T15:25:56.882 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-02T15:25:56.882 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.enable_blob_files: false 2024-04-02T15:25:56.882 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.min_blob_size: 0 2024-04-02T15:25:56.883 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.blob_file_size: 268435456 2024-04-02T15:25:56.883 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-02T15:25:56.883 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-02T15:25:56.883 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-02T15:25:56.883 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-02T15:25:56.883 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-04-02T15:25:56.883 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-04-02T15:25:56.883 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712071556615927, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-04-02T15:25:56.883 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-04-02T15:25:56.883 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-02T15:25:56.883 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712071556616236, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1712071556, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "1a127408-b42c-408b-879b-20256cb37e65", "db_session_id": "QF6OQV7X15MF3GC5A1KE"}} 2024-04-02T15:25:56.883 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-04-02T15:25:56.883 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712071556616593, "job": 1, "event": "recovery_finished"} 2024-04-02T15:25:56.883 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-02T15:25:56.883 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x560d6508aa80 2024-04-02T15:25:56.884 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: DB pointer 0x560d6509a000 2024-04-02T15:25:56.884 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-04-02T15:25:56.884 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-02T15:25:56.884 INFO:journalctl@ceph.mon.b.smithi106.stdout: ** DB Stats ** 2024-04-02T15:25:56.884 INFO:journalctl@ceph.mon.b.smithi106.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-02T15:25:56.884 INFO:journalctl@ceph.mon.b.smithi106.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-02T15:25:56.884 INFO:journalctl@ceph.mon.b.smithi106.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-02T15:25:56.884 INFO:journalctl@ceph.mon.b.smithi106.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-02T15:25:56.884 INFO:journalctl@ceph.mon.b.smithi106.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-02T15:25:56.884 INFO:journalctl@ceph.mon.b.smithi106.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-02T15:25:56.884 INFO:journalctl@ceph.mon.b.smithi106.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-02T15:25:56.884 INFO:journalctl@ceph.mon.b.smithi106.stdout: 2024-04-02T15:25:56.884 INFO:journalctl@ceph.mon.b.smithi106.stdout: ** Compaction Stats [default] ** 2024-04-02T15:25:56.884 INFO:journalctl@ceph.mon.b.smithi106.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-02T15:25:56.884 INFO:journalctl@ceph.mon.b.smithi106.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-02T15:25:56.885 INFO:journalctl@ceph.mon.b.smithi106.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-04-02T15:25:56.885 INFO:journalctl@ceph.mon.b.smithi106.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-04-02T15:25:56.885 INFO:journalctl@ceph.mon.b.smithi106.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-04-02T15:25:56.885 INFO:journalctl@ceph.mon.b.smithi106.stdout: 2024-04-02T15:25:56.885 INFO:journalctl@ceph.mon.b.smithi106.stdout: ** Compaction Stats [default] ** 2024-04-02T15:25:56.885 INFO:journalctl@ceph.mon.b.smithi106.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-02T15:25:56.885 INFO:journalctl@ceph.mon.b.smithi106.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-02T15:25:56.885 INFO:journalctl@ceph.mon.b.smithi106.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-04-02T15:25:56.885 INFO:journalctl@ceph.mon.b.smithi106.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-02T15:25:56.885 INFO:journalctl@ceph.mon.b.smithi106.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-02T15:25:56.885 INFO:journalctl@ceph.mon.b.smithi106.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-02T15:25:56.885 INFO:journalctl@ceph.mon.b.smithi106.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-02T15:25:56.885 INFO:journalctl@ceph.mon.b.smithi106.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-02T15:25:56.885 INFO:journalctl@ceph.mon.b.smithi106.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-02T15:25:56.886 INFO:journalctl@ceph.mon.b.smithi106.stdout: Cumulative compaction: 0.00 GB write, 0.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T15:25:56.886 INFO:journalctl@ceph.mon.b.smithi106.stdout: Interval compaction: 0.00 GB write, 0.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T15:25:56.886 INFO:journalctl@ceph.mon.b.smithi106.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-02T15:25:56.886 INFO:journalctl@ceph.mon.b.smithi106.stdout: 2024-04-02T15:25:56.886 INFO:journalctl@ceph.mon.b.smithi106.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-02T15:25:56.886 INFO:journalctl@ceph.mon.b.smithi106.stdout: 2024-04-02T15:25:56.886 INFO:journalctl@ceph.mon.b.smithi106.stdout: ** Compaction Stats [default] ** 2024-04-02T15:25:56.886 INFO:journalctl@ceph.mon.b.smithi106.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-02T15:25:56.886 INFO:journalctl@ceph.mon.b.smithi106.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-02T15:25:56.886 INFO:journalctl@ceph.mon.b.smithi106.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-04-02T15:25:56.886 INFO:journalctl@ceph.mon.b.smithi106.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-04-02T15:25:56.886 INFO:journalctl@ceph.mon.b.smithi106.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-02T15:25:56.886 INFO:journalctl@ceph.mon.b.smithi106.stdout: 2024-04-02T15:25:56.886 INFO:journalctl@ceph.mon.b.smithi106.stdout: ** Compaction Stats [default] ** 2024-04-02T15:25:56.887 INFO:journalctl@ceph.mon.b.smithi106.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-02T15:25:56.887 INFO:journalctl@ceph.mon.b.smithi106.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-02T15:25:56.887 INFO:journalctl@ceph.mon.b.smithi106.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-04-02T15:25:56.887 INFO:journalctl@ceph.mon.b.smithi106.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-02T15:25:56.887 INFO:journalctl@ceph.mon.b.smithi106.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-02T15:25:56.887 INFO:journalctl@ceph.mon.b.smithi106.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-02T15:25:56.887 INFO:journalctl@ceph.mon.b.smithi106.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-02T15:25:56.887 INFO:journalctl@ceph.mon.b.smithi106.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-02T15:25:56.887 INFO:journalctl@ceph.mon.b.smithi106.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-02T15:25:56.887 INFO:journalctl@ceph.mon.b.smithi106.stdout: Cumulative compaction: 0.00 GB write, 0.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T15:25:56.887 INFO:journalctl@ceph.mon.b.smithi106.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T15:25:56.887 INFO:journalctl@ceph.mon.b.smithi106.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-02T15:25:56.887 INFO:journalctl@ceph.mon.b.smithi106.stdout: 2024-04-02T15:25:56.887 INFO:journalctl@ceph.mon.b.smithi106.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-02T15:25:56.887 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: mon.b does not exist in monmap, will attempt to join an existing cluster 2024-04-02T15:25:56.887 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: using public_addr v2:172.21.15.106:0/0 -> [v2:172.21.15.106:3300/0,v1:172.21.15.106:6789/0] 2024-04-02T15:25:56.888 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: starting mon.b rank -1 at public addrs [v2:172.21.15.106:3300/0,v1:172.21.15.106:6789/0] at bind addrs [v2:172.21.15.106:3300/0,v1:172.21.15.106:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 0400842c-f105-11ee-b647-cb9ed24678a4 2024-04-02T15:25:56.888 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: mon.b@-1(???) e0 preinit fsid 0400842c-f105-11ee-b647-cb9ed24678a4 2024-04-02T15:25:56.888 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: mon.b@-1(synchronizing).mds e1 new map 2024-04-02T15:25:56.888 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: mon.b@-1(synchronizing).mds e1 print_map 2024-04-02T15:25:56.888 INFO:journalctl@ceph.mon.b.smithi106.stdout: e1 2024-04-02T15:25:56.888 INFO:journalctl@ceph.mon.b.smithi106.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-02T15:25:56.888 INFO:journalctl@ceph.mon.b.smithi106.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-02T15:25:56.888 INFO:journalctl@ceph.mon.b.smithi106.stdout: legacy client fscid: -1 2024-04-02T15:25:56.888 INFO:journalctl@ceph.mon.b.smithi106.stdout: 2024-04-02T15:25:56.888 INFO:journalctl@ceph.mon.b.smithi106.stdout: No filesystems configured 2024-04-02T15:25:56.888 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-04-02T15:25:56.888 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-04-02T15:25:56.888 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-04-02T15:25:56.888 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-04-02T15:25:56.888 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-04-02T15:25:56.889 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-04-02T15:25:56.889 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-02T15:25:56.889 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-02T15:25:56.889 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-02T15:25:56.889 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-02T15:25:56.889 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:56.889 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:56.889 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/2590555919' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-02T15:25:56.889 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:25:56.889 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/2590555919' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-02T15:25:56.889 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: osdmap e4: 0 total, 0 up, 0 in 2024-04-02T15:25:56.889 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:56.890 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:25:56.890 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi087:172.21.15.87=a;smithi087:[v2:172.21.15.87:3301,v1:172.21.15.87:6790]=c;smithi106:172.21.15.106=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:25:56.890 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: Saving service mon spec with placement smithi087:172.21.15.87=a;smithi087:[v2:172.21.15.87:3301,v1:172.21.15.87:6790]=c;smithi106:172.21.15.106=b;count:3 2024-04-02T15:25:56.890 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:56.890 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:25:56.890 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:25:56.890 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:25:56.890 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:56.890 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T15:25:56.890 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:25:56.890 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: Deploying daemon mon.c on smithi087 2024-04-02T15:25:56.890 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:25:56.890 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: from='client.? 172.21.15.106:0/1266727001' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T15:25:56.890 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T15:25:56.891 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: mon.a calling monitor election 2024-04-02T15:25:56.891 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:25:56.891 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:25:56.891 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:25:56.891 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:25:56.891 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: mon.c calling monitor election 2024-04-02T15:25:56.891 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:25:56.891 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:25:56.891 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:25:56.891 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:25:56.891 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:25:56.891 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-02T15:25:56.891 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: monmap e2: 2 mons at {a=[v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0],c=[v2:172.21.15.87:3301/0,v1:172.21.15.87:6790/0]} 2024-04-02T15:25:56.891 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: fsmap 2024-04-02T15:25:56.892 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: osdmap e4: 0 total, 0 up, 0 in 2024-04-02T15:25:56.892 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: mgrmap e13: y(active, since 35s) 2024-04-02T15:25:56.892 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: overall HEALTH_OK 2024-04-02T15:25:56.892 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:56.892 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:56.892 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T15:25:56.892 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:25:56.892 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:25:56.892 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: Deploying daemon mon.b on smithi106 2024-04-02T15:25:56.892 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: from='client.? 172.21.15.106:0/1665489746' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T15:25:56.892 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:25:56.892 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:25:56.892 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:25:56.893 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: mon.b@-1(synchronizing).paxosservice(auth 1..5) refresh upgraded, format 0 -> 3 2024-04-02T15:25:56.893 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 ceph-mon[25843]: mon.b@-1(synchronizing) e2 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-04-02T15:25:56.893 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 bash[25803]: d9de5ac6a73e6f45899b23f50d1cac0fe7f5d0af2f64640f7348af83fe702801 2024-04-02T15:25:56.893 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:25:56 smithi106 systemd[1]: Started Ceph mon.b for 0400842c-f105-11ee-b647-cb9ed24678a4. 2024-04-02T15:25:58.240 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-02T15:25:58.240 DEBUG:teuthology.orchestra.run.smithi106:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-04-02T15:26:01.871 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:01 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T15:26:01.872 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:01 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:26:01.872 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:01 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:26:01.872 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:01 smithi106 ceph-mon[25843]: mon.a calling monitor election 2024-04-02T15:26:01.872 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:01 smithi106 ceph-mon[25843]: mon.c calling monitor election 2024-04-02T15:26:01.872 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:01 smithi106 ceph-mon[25843]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:01.872 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:01 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:26:01.872 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:01 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:26:01.872 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:01 smithi106 ceph-mon[25843]: mon.b calling monitor election 2024-04-02T15:26:01.872 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:01 smithi106 ceph-mon[25843]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:01.872 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:01 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:26:01.872 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:01 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:26:01.872 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:01 smithi106 ceph-mon[25843]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:01.873 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:01 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:26:01.873 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:01 smithi106 ceph-mon[25843]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-02T15:26:01.873 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:01 smithi106 ceph-mon[25843]: monmap e3: 3 mons at {a=[v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0],b=[v2:172.21.15.106:3300/0,v1:172.21.15.106:6789/0],c=[v2:172.21.15.87:3301/0,v1:172.21.15.87:6790/0]} 2024-04-02T15:26:01.873 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:01 smithi106 ceph-mon[25843]: fsmap 2024-04-02T15:26:01.873 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:01 smithi106 ceph-mon[25843]: osdmap e4: 0 total, 0 up, 0 in 2024-04-02T15:26:01.873 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:01 smithi106 ceph-mon[25843]: mgrmap e13: y(active, since 44s) 2024-04-02T15:26:01.873 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:01 smithi106 ceph-mon[25843]: overall HEALTH_OK 2024-04-02T15:26:01.873 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:01 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:01.873 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:01 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:26:01.873 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:01 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:01.873 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:01 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:01.873 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:01 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:01.873 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:01 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:26:01.927 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:26:01] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:26:01.928 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T15:26:01.928 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:26:01.928 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:26:01.928 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[29707]: mon.a calling monitor election 2024-04-02T15:26:01.928 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[29707]: mon.c calling monitor election 2024-04-02T15:26:01.928 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[29707]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:01.928 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:26:01.928 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:26:01.928 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[29707]: mon.b calling monitor election 2024-04-02T15:26:01.929 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[29707]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:01.929 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:26:01.929 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:26:01.929 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[29707]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:01.929 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:26:01.929 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[29707]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-02T15:26:01.929 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[29707]: monmap e3: 3 mons at {a=[v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0],b=[v2:172.21.15.106:3300/0,v1:172.21.15.106:6789/0],c=[v2:172.21.15.87:3301/0,v1:172.21.15.87:6790/0]} 2024-04-02T15:26:01.929 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[29707]: fsmap 2024-04-02T15:26:01.929 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[29707]: osdmap e4: 0 total, 0 up, 0 in 2024-04-02T15:26:01.929 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[29707]: mgrmap e13: y(active, since 44s) 2024-04-02T15:26:01.929 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[29707]: overall HEALTH_OK 2024-04-02T15:26:01.929 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:01.929 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:26:01.929 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:01.930 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:01.930 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:01.930 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:26:01.930 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T15:26:01.930 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:26:01.930 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:26:01.930 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[24900]: mon.a calling monitor election 2024-04-02T15:26:01.930 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[24900]: mon.c calling monitor election 2024-04-02T15:26:01.930 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[24900]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:01.930 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:26:01.931 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:26:01.931 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[24900]: mon.b calling monitor election 2024-04-02T15:26:01.931 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[24900]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:01.931 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:26:01.931 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:26:01.931 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[24900]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:01.931 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:26:01.931 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[24900]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-02T15:26:01.931 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[24900]: monmap e3: 3 mons at {a=[v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0],b=[v2:172.21.15.106:3300/0,v1:172.21.15.106:6789/0],c=[v2:172.21.15.87:3301/0,v1:172.21.15.87:6790/0]} 2024-04-02T15:26:01.931 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[24900]: fsmap 2024-04-02T15:26:01.931 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[24900]: osdmap e4: 0 total, 0 up, 0 in 2024-04-02T15:26:01.931 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[24900]: mgrmap e13: y(active, since 44s) 2024-04-02T15:26:01.931 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[24900]: overall HEALTH_OK 2024-04-02T15:26:01.932 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:01.932 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:26:01.932 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:01.932 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:01.932 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:01.932 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:01 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:26:02.478 INFO:teuthology.orchestra.run.smithi106.stdout: 2024-04-02T15:26:02.478 INFO:teuthology.orchestra.run.smithi106.stdout:{"epoch":3,"fsid":"0400842c-f105-11ee-b647-cb9ed24678a4","modified":"2024-04-02T15:25:56.645912Z","created":"2024-04-02T15:24:21.862790Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:3300","nonce":0},{"type":"v1","addr":"172.21.15.87:6789","nonce":0}]},"addr":"172.21.15.87:6789/0","public_addr":"172.21.15.87:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:3301","nonce":0},{"type":"v1","addr":"172.21.15.87:6790","nonce":0}]},"addr":"172.21.15.87:6790/0","public_addr":"172.21.15.87:6790/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:3300","nonce":0},{"type":"v1","addr":"172.21.15.106:6789","nonce":0}]},"addr":"172.21.15.106:6789/0","public_addr":"172.21.15.106:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-04-02T15:26:02.480 INFO:teuthology.orchestra.run.smithi106.stderr:dumped monmap epoch 3 2024-04-02T15:26:03.239 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:02 smithi087 ceph-mon[24900]: Updating smithi087:/etc/ceph/ceph.conf 2024-04-02T15:26:03.239 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:02 smithi087 ceph-mon[24900]: Updating smithi106:/etc/ceph/ceph.conf 2024-04-02T15:26:03.239 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:02 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:03.240 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:02 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:03.240 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:02 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:03.240 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:02 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:03.240 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:02 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T15:26:03.240 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:02 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T15:26:03.240 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:02 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:03.240 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:02 smithi087 ceph-mon[24900]: from='client.? 172.21.15.106:0/4282331670' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T15:26:03.240 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:02 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:26:03.240 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:02 smithi087 ceph-mon[29707]: Updating smithi087:/etc/ceph/ceph.conf 2024-04-02T15:26:03.240 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:02 smithi087 ceph-mon[29707]: Updating smithi106:/etc/ceph/ceph.conf 2024-04-02T15:26:03.240 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:02 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:03.240 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:02 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:03.240 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:02 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:03.241 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:02 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:03.241 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:02 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T15:26:03.241 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:02 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T15:26:03.241 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:02 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:03.241 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:02 smithi087 ceph-mon[29707]: from='client.? 172.21.15.106:0/4282331670' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T15:26:03.241 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:02 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:26:03.264 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:02 smithi106 ceph-mon[25843]: Updating smithi087:/etc/ceph/ceph.conf 2024-04-02T15:26:03.264 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:02 smithi106 ceph-mon[25843]: Updating smithi106:/etc/ceph/ceph.conf 2024-04-02T15:26:03.264 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:02 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:03.264 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:02 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:03.264 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:02 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:03.265 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:02 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:03.265 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:02 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T15:26:03.265 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:02 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T15:26:03.265 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:02 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:03.265 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:02 smithi106 ceph-mon[25843]: from='client.? 172.21.15.106:0/4282331670' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T15:26:03.265 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:02 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:26:04.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:03 smithi106 ceph-mon[25843]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-02T15:26:04.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:03 smithi106 ceph-mon[25843]: Reconfiguring mon.a (unknown last config time)... 2024-04-02T15:26:04.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:03 smithi106 ceph-mon[25843]: Reconfiguring daemon mon.a on smithi087 2024-04-02T15:26:04.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:03 smithi106 ceph-mon[25843]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:04.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:03 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:04.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:03 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T15:26:04.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:03 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T15:26:04.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:03 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:04.168 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-04-02T15:26:04.169 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph config generate-minimal-conf 2024-04-02T15:26:04.194 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:26:03 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:26:03] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:26:04.195 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:03 smithi087 ceph-mon[29707]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-02T15:26:04.195 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:03 smithi087 ceph-mon[29707]: Reconfiguring mon.a (unknown last config time)... 2024-04-02T15:26:04.195 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:03 smithi087 ceph-mon[29707]: Reconfiguring daemon mon.a on smithi087 2024-04-02T15:26:04.195 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:03 smithi087 ceph-mon[29707]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:04.195 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:03 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:04.195 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:03 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T15:26:04.195 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:03 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T15:26:04.195 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:03 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:04.195 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:03 smithi087 ceph-mon[24900]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-02T15:26:04.195 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:03 smithi087 ceph-mon[24900]: Reconfiguring mon.a (unknown last config time)... 2024-04-02T15:26:04.195 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:03 smithi087 ceph-mon[24900]: Reconfiguring daemon mon.a on smithi087 2024-04-02T15:26:04.196 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:03 smithi087 ceph-mon[24900]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:04.196 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:03 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:04.196 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:03 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T15:26:04.196 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:03 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T15:26:04.196 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:03 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:05.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:04 smithi106 ceph-mon[25843]: Reconfiguring mon.c (monmap changed)... 2024-04-02T15:26:05.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:04 smithi106 ceph-mon[25843]: Reconfiguring daemon mon.c on smithi087 2024-04-02T15:26:05.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:04 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:05.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:04 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:05.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:04 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:05.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:04 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:05.119 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:26:04 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:26:04] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:26:05.119 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:26:05 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:26:05] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:26:05.119 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:04 smithi087 ceph-mon[29707]: Reconfiguring mon.c (monmap changed)... 2024-04-02T15:26:05.119 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:04 smithi087 ceph-mon[29707]: Reconfiguring daemon mon.c on smithi087 2024-04-02T15:26:05.119 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:04 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:05.119 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:04 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:05.119 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:04 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:05.119 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:04 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:05.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:04 smithi087 ceph-mon[24900]: Reconfiguring mon.c (monmap changed)... 2024-04-02T15:26:05.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:04 smithi087 ceph-mon[24900]: Reconfiguring daemon mon.c on smithi087 2024-04-02T15:26:05.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:04 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:05.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:04 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:05.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:04 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:05.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:04 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:06.101 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:05 smithi106 ceph-mon[25843]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:06.101 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:05 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:06.101 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:05 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:06.101 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:05 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:06.101 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:05 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T15:26:06.101 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:05 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T15:26:06.102 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:05 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:06.304 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:05 smithi087 ceph-mon[24900]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:06.305 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:05 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:06.305 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:05 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:06.305 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:05 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:06.305 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:05 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T15:26:06.305 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:05 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T15:26:06.305 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:05 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:06.305 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:05 smithi087 ceph-mon[29707]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:06.305 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:05 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:06.305 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:05 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:06.305 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:05 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:06.305 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:05 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T15:26:06.305 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:05 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T15:26:06.305 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:05 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:07.100 INFO:teuthology.orchestra.run.smithi087.stdout:# minimal ceph.conf for 0400842c-f105-11ee-b647-cb9ed24678a4 2024-04-02T15:26:07.100 INFO:teuthology.orchestra.run.smithi087.stdout:[global] 2024-04-02T15:26:07.100 INFO:teuthology.orchestra.run.smithi087.stdout: fsid = 0400842c-f105-11ee-b647-cb9ed24678a4 2024-04-02T15:26:07.100 INFO:teuthology.orchestra.run.smithi087.stdout: mon_host = [v2:172.21.15.87:3300/0,v1:172.21.15.87:6789/0] [v2:172.21.15.106:3300/0,v1:172.21.15.106:6789/0] [v2:172.21.15.87:3301/0,v1:172.21.15.87:6790/0] 2024-04-02T15:26:07.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:06 smithi106 ceph-mon[25843]: Reconfiguring mon.b (monmap changed)... 2024-04-02T15:26:07.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:06 smithi106 ceph-mon[25843]: Reconfiguring daemon mon.b on smithi106 2024-04-02T15:26:07.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:06 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:07.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:06 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:26:07.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:06 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:07.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:06 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:26:07.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:06 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:07.138 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:06 smithi087 ceph-mon[29707]: Reconfiguring mon.b (monmap changed)... 2024-04-02T15:26:07.138 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:06 smithi087 ceph-mon[29707]: Reconfiguring daemon mon.b on smithi106 2024-04-02T15:26:07.138 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:06 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:07.138 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:06 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:26:07.138 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:06 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:07.138 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:06 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:26:07.138 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:06 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:07.138 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:06 smithi087 ceph-mon[24900]: Reconfiguring mon.b (monmap changed)... 2024-04-02T15:26:07.139 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:06 smithi087 ceph-mon[24900]: Reconfiguring daemon mon.b on smithi106 2024-04-02T15:26:07.139 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:06 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:07.139 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:06 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:26:07.139 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:06 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:07.139 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:06 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:26:07.139 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:06 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:08.277 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:26:08 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:26:08] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:26:08.277 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:07 smithi087 ceph-mon[29707]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:08.277 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:07 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/1105116787' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:08.278 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:07 smithi087 ceph-mon[24900]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:08.278 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:07 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/1105116787' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:08.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:07 smithi106 ceph-mon[25843]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:08.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:07 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/1105116787' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:08.368 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-04-02T15:26:08.368 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2024-04-02T15:26:08.369 DEBUG:teuthology.orchestra.run.smithi087:> sudo dd of=/etc/ceph/ceph.conf 2024-04-02T15:26:08.438 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2024-04-02T15:26:08.438 DEBUG:teuthology.orchestra.run.smithi087:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-02T15:26:08.505 DEBUG:teuthology.orchestra.run.smithi106:> set -ex 2024-04-02T15:26:08.505 DEBUG:teuthology.orchestra.run.smithi106:> sudo dd of=/etc/ceph/ceph.conf 2024-04-02T15:26:08.540 DEBUG:teuthology.orchestra.run.smithi106:> set -ex 2024-04-02T15:26:08.540 DEBUG:teuthology.orchestra.run.smithi106:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-02T15:26:08.612 INFO:tasks.cephadm:Adding mgr.y on smithi087 2024-04-02T15:26:08.612 INFO:tasks.cephadm:Adding mgr.x on smithi106 2024-04-02T15:26:08.612 DEBUG:teuthology.orchestra.run.smithi106:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph orch apply mgr '2;smithi087=y;smithi106=x' 2024-04-02T15:26:09.265 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:09 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:09.265 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:09 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:09.265 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:09 smithi106 ceph-mon[25843]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:09.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:09 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:09.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:09 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:09.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:09 smithi087 ceph-mon[24900]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:09.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:09 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:09.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:09 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:09.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:09 smithi087 ceph-mon[29707]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:10.491 INFO:teuthology.orchestra.run.smithi106.stdout:Scheduled mgr update... 2024-04-02T15:26:11.072 DEBUG:teuthology.orchestra.run.smithi106:mgr.x> sudo journalctl -f -n 0 -u ceph-0400842c-f105-11ee-b647-cb9ed24678a4@mgr.x.service 2024-04-02T15:26:11.074 INFO:tasks.cephadm:Deploying OSDs... 2024-04-02T15:26:11.074 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2024-04-02T15:26:11.074 DEBUG:teuthology.orchestra.run.smithi087:> dd if=/scratch_devs of=/dev/stdout 2024-04-02T15:26:11.093 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-02T15:26:11.093 DEBUG:teuthology.orchestra.run.smithi087:> stat /dev/vg_nvme/lv_1 2024-04-02T15:26:11.151 INFO:teuthology.orchestra.run.smithi087.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-02T15:26:11.152 INFO:teuthology.orchestra.run.smithi087.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T15:26:11.152 INFO:teuthology.orchestra.run.smithi087.stdout:Device: 5h/5d Inode: 667 Links: 1 2024-04-02T15:26:11.152 INFO:teuthology.orchestra.run.smithi087.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T15:26:11.152 INFO:teuthology.orchestra.run.smithi087.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T15:26:11.152 INFO:teuthology.orchestra.run.smithi087.stdout:Access: 2024-04-02 15:26:07.115327821 +0000 2024-04-02T15:26:11.152 INFO:teuthology.orchestra.run.smithi087.stdout:Modify: 2024-04-02 15:23:29.852462664 +0000 2024-04-02T15:26:11.152 INFO:teuthology.orchestra.run.smithi087.stdout:Change: 2024-04-02 15:23:29.852462664 +0000 2024-04-02T15:26:11.152 INFO:teuthology.orchestra.run.smithi087.stdout: Birth: 2024-04-02 15:23:29.852462664 +0000 2024-04-02T15:26:11.152 DEBUG:teuthology.orchestra.run.smithi087:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-02T15:26:11.220 INFO:teuthology.orchestra.run.smithi087.stderr:1+0 records in 2024-04-02T15:26:11.221 INFO:teuthology.orchestra.run.smithi087.stderr:1+0 records out 2024-04-02T15:26:11.221 INFO:teuthology.orchestra.run.smithi087.stderr:512 bytes copied, 0.000230094 s, 2.2 MB/s 2024-04-02T15:26:11.222 DEBUG:teuthology.orchestra.run.smithi087:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-02T15:26:11.280 DEBUG:teuthology.orchestra.run.smithi087:> stat /dev/vg_nvme/lv_2 2024-04-02T15:26:11.337 INFO:teuthology.orchestra.run.smithi087.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-02T15:26:11.337 INFO:teuthology.orchestra.run.smithi087.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T15:26:11.337 INFO:teuthology.orchestra.run.smithi087.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-04-02T15:26:11.337 INFO:teuthology.orchestra.run.smithi087.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T15:26:11.337 INFO:teuthology.orchestra.run.smithi087.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T15:26:11.337 INFO:teuthology.orchestra.run.smithi087.stdout:Access: 2024-04-02 15:26:07.115327821 +0000 2024-04-02T15:26:11.337 INFO:teuthology.orchestra.run.smithi087.stdout:Modify: 2024-04-02 15:23:30.205454660 +0000 2024-04-02T15:26:11.337 INFO:teuthology.orchestra.run.smithi087.stdout:Change: 2024-04-02 15:23:30.205454660 +0000 2024-04-02T15:26:11.337 INFO:teuthology.orchestra.run.smithi087.stdout: Birth: 2024-04-02 15:23:30.205454660 +0000 2024-04-02T15:26:11.337 DEBUG:teuthology.orchestra.run.smithi087:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-02T15:26:11.403 INFO:teuthology.orchestra.run.smithi087.stderr:1+0 records in 2024-04-02T15:26:11.404 INFO:teuthology.orchestra.run.smithi087.stderr:1+0 records out 2024-04-02T15:26:11.404 INFO:teuthology.orchestra.run.smithi087.stderr:512 bytes copied, 0.000227606 s, 2.2 MB/s 2024-04-02T15:26:11.405 DEBUG:teuthology.orchestra.run.smithi087:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-02T15:26:11.462 DEBUG:teuthology.orchestra.run.smithi087:> stat /dev/vg_nvme/lv_3 2024-04-02T15:26:11.518 INFO:teuthology.orchestra.run.smithi087.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-02T15:26:11.518 INFO:teuthology.orchestra.run.smithi087.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T15:26:11.518 INFO:teuthology.orchestra.run.smithi087.stdout:Device: 5h/5d Inode: 704 Links: 1 2024-04-02T15:26:11.518 INFO:teuthology.orchestra.run.smithi087.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T15:26:11.518 INFO:teuthology.orchestra.run.smithi087.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T15:26:11.518 INFO:teuthology.orchestra.run.smithi087.stdout:Access: 2024-04-02 15:26:07.115327821 +0000 2024-04-02T15:26:11.518 INFO:teuthology.orchestra.run.smithi087.stdout:Modify: 2024-04-02 15:23:30.544446973 +0000 2024-04-02T15:26:11.518 INFO:teuthology.orchestra.run.smithi087.stdout:Change: 2024-04-02 15:23:30.544446973 +0000 2024-04-02T15:26:11.519 INFO:teuthology.orchestra.run.smithi087.stdout: Birth: 2024-04-02 15:23:30.544446973 +0000 2024-04-02T15:26:11.519 DEBUG:teuthology.orchestra.run.smithi087:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-02T15:26:11.560 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:11 smithi106 ceph-mon[25843]: from='client.24113 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi087=y;smithi106=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:26:11.560 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:11 smithi106 ceph-mon[25843]: Saving service mgr spec with placement smithi087=y;smithi106=x;count:2 2024-04-02T15:26:11.560 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:11 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:11.560 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:11 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:26:11.560 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:11 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:11.560 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:11 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:26:11.560 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:11 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:11.560 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:11 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T15:26:11.560 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:11 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-02T15:26:11.560 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:11 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T15:26:11.561 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:11 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:11.561 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:11 smithi106 ceph-mon[25843]: Deploying daemon mgr.x on smithi106 2024-04-02T15:26:11.561 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:11 smithi106 ceph-mon[25843]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:11.586 INFO:teuthology.orchestra.run.smithi087.stderr:1+0 records in 2024-04-02T15:26:11.586 INFO:teuthology.orchestra.run.smithi087.stderr:1+0 records out 2024-04-02T15:26:11.586 INFO:teuthology.orchestra.run.smithi087.stderr:512 bytes copied, 0.000193806 s, 2.6 MB/s 2024-04-02T15:26:11.587 DEBUG:teuthology.orchestra.run.smithi087:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-02T15:26:11.643 DEBUG:teuthology.orchestra.run.smithi087:> stat /dev/vg_nvme/lv_4 2024-04-02T15:26:11.699 INFO:teuthology.orchestra.run.smithi087.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-02T15:26:11.699 INFO:teuthology.orchestra.run.smithi087.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T15:26:11.699 INFO:teuthology.orchestra.run.smithi087.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-04-02T15:26:11.699 INFO:teuthology.orchestra.run.smithi087.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T15:26:11.699 INFO:teuthology.orchestra.run.smithi087.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T15:26:11.700 INFO:teuthology.orchestra.run.smithi087.stdout:Access: 2024-04-02 15:26:07.115327821 +0000 2024-04-02T15:26:11.700 INFO:teuthology.orchestra.run.smithi087.stdout:Modify: 2024-04-02 15:23:30.878439400 +0000 2024-04-02T15:26:11.700 INFO:teuthology.orchestra.run.smithi087.stdout:Change: 2024-04-02 15:23:30.878439400 +0000 2024-04-02T15:26:11.700 INFO:teuthology.orchestra.run.smithi087.stdout: Birth: 2024-04-02 15:23:30.878439400 +0000 2024-04-02T15:26:11.700 DEBUG:teuthology.orchestra.run.smithi087:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-02T15:26:11.764 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:11 smithi087 ceph-mon[24900]: from='client.24113 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi087=y;smithi106=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:26:11.764 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:11 smithi087 ceph-mon[24900]: Saving service mgr spec with placement smithi087=y;smithi106=x;count:2 2024-04-02T15:26:11.764 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:11 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:11.764 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:11 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:26:11.764 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:11 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:11.764 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:11 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:26:11.764 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:11 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:11.764 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:11 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T15:26:11.764 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:11 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-02T15:26:11.764 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:11 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T15:26:11.765 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:11 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:11.765 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:11 smithi087 ceph-mon[24900]: Deploying daemon mgr.x on smithi106 2024-04-02T15:26:11.765 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:11 smithi087 ceph-mon[24900]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:11.765 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:11 smithi087 ceph-mon[29707]: from='client.24113 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi087=y;smithi106=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:26:11.765 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:11 smithi087 ceph-mon[29707]: Saving service mgr spec with placement smithi087=y;smithi106=x;count:2 2024-04-02T15:26:11.765 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:11 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:11.765 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:11 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:26:11.765 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:11 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:11.765 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:11 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:26:11.765 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:11 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:11.765 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:11 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T15:26:11.765 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:11 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-02T15:26:11.765 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:11 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T15:26:11.766 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:11 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:11.766 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:11 smithi087 ceph-mon[29707]: Deploying daemon mgr.x on smithi106 2024-04-02T15:26:11.766 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:11 smithi087 ceph-mon[29707]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:11.766 INFO:teuthology.orchestra.run.smithi087.stderr:1+0 records in 2024-04-02T15:26:11.766 INFO:teuthology.orchestra.run.smithi087.stderr:1+0 records out 2024-04-02T15:26:11.766 INFO:teuthology.orchestra.run.smithi087.stderr:512 bytes copied, 0.000204055 s, 2.5 MB/s 2024-04-02T15:26:11.767 DEBUG:teuthology.orchestra.run.smithi087:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-02T15:26:11.825 DEBUG:teuthology.orchestra.run.smithi106:> set -ex 2024-04-02T15:26:11.825 DEBUG:teuthology.orchestra.run.smithi106:> dd if=/scratch_devs of=/dev/stdout 2024-04-02T15:26:11.840 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-02T15:26:11.840 DEBUG:teuthology.orchestra.run.smithi106:> stat /dev/vg_nvme/lv_1 2024-04-02T15:26:11.895 INFO:teuthology.orchestra.run.smithi106.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-02T15:26:11.896 INFO:teuthology.orchestra.run.smithi106.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T15:26:11.896 INFO:teuthology.orchestra.run.smithi106.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-04-02T15:26:11.896 INFO:teuthology.orchestra.run.smithi106.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T15:26:11.896 INFO:teuthology.orchestra.run.smithi106.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T15:26:11.896 INFO:teuthology.orchestra.run.smithi106.stdout:Access: 2024-04-02 15:26:04.110279811 +0000 2024-04-02T15:26:11.896 INFO:teuthology.orchestra.run.smithi106.stdout:Modify: 2024-04-02 15:23:29.859250897 +0000 2024-04-02T15:26:11.896 INFO:teuthology.orchestra.run.smithi106.stdout:Change: 2024-04-02 15:23:29.859250897 +0000 2024-04-02T15:26:11.896 INFO:teuthology.orchestra.run.smithi106.stdout: Birth: 2024-04-02 15:23:29.859250897 +0000 2024-04-02T15:26:11.896 DEBUG:teuthology.orchestra.run.smithi106:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-02T15:26:11.962 INFO:teuthology.orchestra.run.smithi106.stderr:1+0 records in 2024-04-02T15:26:11.962 INFO:teuthology.orchestra.run.smithi106.stderr:1+0 records out 2024-04-02T15:26:11.962 INFO:teuthology.orchestra.run.smithi106.stderr:512 bytes copied, 0.000186047 s, 2.8 MB/s 2024-04-02T15:26:11.963 DEBUG:teuthology.orchestra.run.smithi106:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-02T15:26:12.018 DEBUG:teuthology.orchestra.run.smithi106:> stat /dev/vg_nvme/lv_2 2024-04-02T15:26:12.074 INFO:teuthology.orchestra.run.smithi106.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-02T15:26:12.074 INFO:teuthology.orchestra.run.smithi106.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T15:26:12.074 INFO:teuthology.orchestra.run.smithi106.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-04-02T15:26:12.074 INFO:teuthology.orchestra.run.smithi106.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T15:26:12.074 INFO:teuthology.orchestra.run.smithi106.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T15:26:12.074 INFO:teuthology.orchestra.run.smithi106.stdout:Access: 2024-04-02 15:26:04.110279811 +0000 2024-04-02T15:26:12.074 INFO:teuthology.orchestra.run.smithi106.stdout:Modify: 2024-04-02 15:23:30.203242489 +0000 2024-04-02T15:26:12.074 INFO:teuthology.orchestra.run.smithi106.stdout:Change: 2024-04-02 15:23:30.203242489 +0000 2024-04-02T15:26:12.074 INFO:teuthology.orchestra.run.smithi106.stdout: Birth: 2024-04-02 15:23:30.203242489 +0000 2024-04-02T15:26:12.075 DEBUG:teuthology.orchestra.run.smithi106:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-02T15:26:12.139 INFO:teuthology.orchestra.run.smithi106.stderr:1+0 records in 2024-04-02T15:26:12.139 INFO:teuthology.orchestra.run.smithi106.stderr:1+0 records out 2024-04-02T15:26:12.139 INFO:teuthology.orchestra.run.smithi106.stderr:512 bytes copied, 0.000175337 s, 2.9 MB/s 2024-04-02T15:26:12.140 DEBUG:teuthology.orchestra.run.smithi106:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-02T15:26:12.194 DEBUG:teuthology.orchestra.run.smithi106:> stat /dev/vg_nvme/lv_3 2024-04-02T15:26:12.250 INFO:teuthology.orchestra.run.smithi106.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-02T15:26:12.250 INFO:teuthology.orchestra.run.smithi106.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T15:26:12.250 INFO:teuthology.orchestra.run.smithi106.stdout:Device: 5h/5d Inode: 703 Links: 1 2024-04-02T15:26:12.250 INFO:teuthology.orchestra.run.smithi106.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T15:26:12.250 INFO:teuthology.orchestra.run.smithi106.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T15:26:12.250 INFO:teuthology.orchestra.run.smithi106.stdout:Access: 2024-04-02 15:26:04.110279811 +0000 2024-04-02T15:26:12.250 INFO:teuthology.orchestra.run.smithi106.stdout:Modify: 2024-04-02 15:23:30.538234301 +0000 2024-04-02T15:26:12.250 INFO:teuthology.orchestra.run.smithi106.stdout:Change: 2024-04-02 15:23:30.538234301 +0000 2024-04-02T15:26:12.250 INFO:teuthology.orchestra.run.smithi106.stdout: Birth: 2024-04-02 15:23:30.538234301 +0000 2024-04-02T15:26:12.251 DEBUG:teuthology.orchestra.run.smithi106:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-02T15:26:12.386 INFO:teuthology.orchestra.run.smithi106.stderr:1+0 records in 2024-04-02T15:26:12.386 INFO:teuthology.orchestra.run.smithi106.stderr:1+0 records out 2024-04-02T15:26:12.386 INFO:teuthology.orchestra.run.smithi106.stderr:512 bytes copied, 0.000201022 s, 2.5 MB/s 2024-04-02T15:26:12.387 DEBUG:teuthology.orchestra.run.smithi106:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-02T15:26:12.402 DEBUG:teuthology.orchestra.run.smithi106:> stat /dev/vg_nvme/lv_4 2024-04-02T15:26:12.458 INFO:teuthology.orchestra.run.smithi106.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-02T15:26:12.458 INFO:teuthology.orchestra.run.smithi106.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T15:26:12.458 INFO:teuthology.orchestra.run.smithi106.stdout:Device: 5h/5d Inode: 725 Links: 1 2024-04-02T15:26:12.458 INFO:teuthology.orchestra.run.smithi106.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T15:26:12.458 INFO:teuthology.orchestra.run.smithi106.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T15:26:12.458 INFO:teuthology.orchestra.run.smithi106.stdout:Access: 2024-04-02 15:26:12.349083285 +0000 2024-04-02T15:26:12.458 INFO:teuthology.orchestra.run.smithi106.stdout:Modify: 2024-04-02 15:23:30.883225869 +0000 2024-04-02T15:26:12.459 INFO:teuthology.orchestra.run.smithi106.stdout:Change: 2024-04-02 15:23:30.883225869 +0000 2024-04-02T15:26:12.459 INFO:teuthology.orchestra.run.smithi106.stdout: Birth: 2024-04-02 15:23:30.883225869 +0000 2024-04-02T15:26:12.459 DEBUG:teuthology.orchestra.run.smithi106:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-02T15:26:12.555 INFO:teuthology.orchestra.run.smithi106.stderr:1+0 records in 2024-04-02T15:26:12.555 INFO:teuthology.orchestra.run.smithi106.stderr:1+0 records out 2024-04-02T15:26:12.555 INFO:teuthology.orchestra.run.smithi106.stderr:512 bytes copied, 0.000195888 s, 2.6 MB/s 2024-04-02T15:26:12.556 DEBUG:teuthology.orchestra.run.smithi106:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-02T15:26:12.571 INFO:tasks.cephadm:Deploying osd.0 on smithi087 with /dev/vg_nvme/lv_4... 2024-04-02T15:26:12.571 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-02T15:26:12.789 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:26:12 smithi106 systemd[1]: Starting Ceph mgr.x for 0400842c-f105-11ee-b647-cb9ed24678a4... 2024-04-02T15:26:13.091 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:26:12 smithi106 podman[27370]: 2024-04-02 15:26:12.794091039 +0000 UTC m=+0.114052597 container create 7d9b02b111e79df1f5509fac2ab5144ccffc9a5aa8e04b4e580e7087217e460e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, io.openshift.tags=base centos centos-stream, distribution-scope=public, vendor=Red Hat, Inc., release=754, maintainer=Guillaume Abrioux , io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, vcs-type=git, version=8, GIT_CLEAN=True) 2024-04-02T15:26:13.092 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:26:12 smithi106 podman[27370]: 2024-04-02 15:26:12.698142259 +0000 UTC m=+0.018103830 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-02T15:26:13.092 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:26:12 smithi106 podman[27370]: 2024-04-02 15:26:12.996234678 +0000 UTC m=+0.316196248 container init 7d9b02b111e79df1f5509fac2ab5144ccffc9a5aa8e04b4e580e7087217e460e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x, name=centos-stream, architecture=x86_64, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, io.buildah.version=1.19.8, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, RELEASE=HEAD, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0) 2024-04-02T15:26:13.092 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:26:12 smithi106 podman[27370]: 2024-04-02 15:26:12.99891459 +0000 UTC m=+0.318876154 container start 7d9b02b111e79df1f5509fac2ab5144ccffc9a5aa8e04b4e580e7087217e460e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, vcs-type=git, version=8, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, release=754, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, distribution-scope=public) 2024-04-02T15:26:13.092 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:26:13 smithi106 ceph-mgr[27389]: -- 172.21.15.106:0/2273247924 <== mon.2 v2:172.21.15.106:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x555bb9d94340 con 0x555bb9d6cc00 2024-04-02T15:26:13.092 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:26:13 smithi106 bash[27370]: 7d9b02b111e79df1f5509fac2ab5144ccffc9a5aa8e04b4e580e7087217e460e 2024-04-02T15:26:13.365 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:26:13 smithi106 systemd[1]: Started Ceph mgr.x for 0400842c-f105-11ee-b647-cb9ed24678a4. 2024-04-02T15:26:13.747 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:26:13 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:26:13.476+0000 7fa0c4380000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-02T15:26:13.747 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:26:13 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:26:13.564+0000 7fa0c4380000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-02T15:26:14.037 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:13 smithi087 ceph-mon[29707]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:14.037 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:13 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:14.037 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:13 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:26:14.037 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:13 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:14.037 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:13 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:26:14.038 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:13 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:14.038 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:13 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T15:26:14.038 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:13 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T15:26:14.038 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:13 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:14.038 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:13 smithi087 ceph-mon[24900]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:14.038 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:13 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:14.038 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:13 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:26:14.038 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:13 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:14.038 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:13 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:26:14.038 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:13 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:14.038 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:13 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T15:26:14.039 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:13 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T15:26:14.039 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:13 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:14.044 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:13 smithi106 ceph-mon[25843]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:14.044 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:13 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:14.044 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:13 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:26:14.044 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:13 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:14.044 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:13 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:26:14.044 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:13 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:14.044 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:13 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T15:26:14.044 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:13 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T15:26:14.044 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:13 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:14.303 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:26:14 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:26:14.181+0000 7fa0c4380000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-02T15:26:14.303 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:26:14 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:26:14.269+0000 7fa0c4380000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-02T15:26:14.590 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:26:14 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:26:14.365+0000 7fa0c4380000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-02T15:26:14.590 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:26:14 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:26:14.515+0000 7fa0c4380000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-02T15:26:14.590 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:26:14 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:26:14.594+0000 7fa0c4380000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-02T15:26:14.865 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:26:14 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:26:14.703+0000 7fa0c4380000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-02T15:26:14.865 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:26:14 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:26:14.794+0000 7fa0c4380000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-02T15:26:15.187 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:14 smithi087 ceph-mon[29707]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-02T15:26:15.187 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:14 smithi087 ceph-mon[29707]: Reconfiguring mgr.y (unknown last config time)... 2024-04-02T15:26:15.187 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:14 smithi087 ceph-mon[29707]: Reconfiguring daemon mgr.y on smithi087 2024-04-02T15:26:15.187 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:14 smithi087 ceph-mon[24900]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-02T15:26:15.187 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:14 smithi087 ceph-mon[24900]: Reconfiguring mgr.y (unknown last config time)... 2024-04-02T15:26:15.187 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:14 smithi087 ceph-mon[24900]: Reconfiguring daemon mgr.y on smithi087 2024-04-02T15:26:15.360 INFO:teuthology.orchestra.run.smithi087.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-02T15:26:15.360 INFO:teuthology.orchestra.run.smithi087.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-02T15:26:15.360 INFO:teuthology.orchestra.run.smithi087.stderr: stderr: 10+0 records in 2024-04-02T15:26:15.360 INFO:teuthology.orchestra.run.smithi087.stderr:10+0 records out 2024-04-02T15:26:15.360 INFO:teuthology.orchestra.run.smithi087.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0115019 s, 912 MB/s 2024-04-02T15:26:15.360 INFO:teuthology.orchestra.run.smithi087.stderr:--> Zapping successful for: 2024-04-02T15:26:15.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:14 smithi106 ceph-mon[25843]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-02T15:26:15.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:14 smithi106 ceph-mon[25843]: Reconfiguring mgr.y (unknown last config time)... 2024-04-02T15:26:15.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:14 smithi106 ceph-mon[25843]: Reconfiguring daemon mgr.y on smithi087 2024-04-02T15:26:15.365 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:26:14 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:26:14.960+0000 7fa0c4380000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-02T15:26:15.365 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:26:15 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:26:15.045+0000 7fa0c4380000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-02T15:26:16.081 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi087:vg_nvme/lv_4 2024-04-02T15:26:16.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:15 smithi106 ceph-mon[25843]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:16.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:15 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:16.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:15 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:26:16.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:15 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:16.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:15 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:26:16.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:15 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:16.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:15 smithi087 ceph-mon[24900]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:16.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:15 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:16.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:15 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:26:16.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:15 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:16.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:15 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:26:16.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:15 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:16.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:15 smithi087 ceph-mon[29707]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:16.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:15 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:16.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:15 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:26:16.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:15 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:16.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:15 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:26:16.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:15 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:16.380 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:26:16 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:26:16.168+0000 7fa0c4380000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-02T15:26:16.380 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:26:16 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:26:16.296+0000 7fa0c4380000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-02T15:26:16.652 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:26:16 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:26:16] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:26:16.719 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:26:16 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:26:16.491+0000 7fa0c4380000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-02T15:26:16.719 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:26:16 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:26:16.577+0000 7fa0c4380000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-02T15:26:16.720 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:26:16 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:26:16.723+0000 7fa0c4380000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-02T15:26:17.055 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:16 smithi106 ceph-mon[25843]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-02T15:26:17.056 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:16 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:17.056 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:16 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:17.056 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:16 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:17.056 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:16 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:26:17.056 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:16 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:17.056 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:16 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:26:17.056 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:16 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:17.056 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:16 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:26:17.056 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:16 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:26:17.056 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:16 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:17.058 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:26:16 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:26:16] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:26:17.058 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:16 smithi087 ceph-mon[29707]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-02T15:26:17.059 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:16 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:17.059 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:16 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:17.059 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:16 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:17.059 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:16 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:26:17.059 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:16 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:17.059 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:16 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:26:17.059 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:16 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:17.059 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:16 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:26:17.059 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:16 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:26:17.059 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:16 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:17.059 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:16 smithi087 ceph-mon[24900]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-02T15:26:17.059 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:16 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:17.059 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:16 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:17.059 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:16 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:17.060 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:16 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:26:17.060 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:16 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:17.060 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:16 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:26:17.060 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:16 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:17.060 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:16 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:26:17.060 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:16 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:26:17.060 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:16 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:17.357 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:26:17 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:26:17.059+0000 7fa0c4380000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-02T15:26:17.357 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:26:17 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:26:17.279+0000 7fa0c4380000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-02T15:26:17.357 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:26:17 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:26:17.360+0000 7fa0c4380000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-02T15:26:17.615 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:26:17 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:26:17.530+0000 7fa0c4380000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-02T15:26:18.183 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:17 smithi087 ceph-mon[24900]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:18.183 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:17 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T15:26:18.183 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:17 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T15:26:18.183 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:17 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:18.184 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:17 smithi087 ceph-mon[29707]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:18.184 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:17 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T15:26:18.184 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:17 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T15:26:18.184 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:17 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:18.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:17 smithi106 ceph-mon[25843]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:18.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:17 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T15:26:18.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:17 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T15:26:18.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:17 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:18.728 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:26:18 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:26:18.443+0000 7fa0c4380000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-02T15:26:18.728 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:26:18 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:26:18.732+0000 7fa0c4380000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-02T15:26:19.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:18 smithi106 ceph-mon[25843]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi087:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:26:19.115 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:26:18 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:26:18.824+0000 7fa0c4380000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-02T15:26:19.115 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:26:18 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:26:18.972+0000 7fa0c4380000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-02T15:26:19.212 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:18 smithi087 ceph-mon[24900]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi087:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:26:19.212 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:18 smithi087 ceph-mon[29707]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi087:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:26:20.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:19 smithi106 ceph-mon[25843]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:20.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:19 smithi106 ceph-mon[25843]: Standby manager daemon x started 2024-04-02T15:26:20.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:19 smithi106 ceph-mon[25843]: from='mgr.? 172.21.15.106:0/2350971553' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-02T15:26:20.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:19 smithi106 ceph-mon[25843]: from='mgr.? 172.21.15.106:0/2350971553' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T15:26:20.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:19 smithi106 ceph-mon[25843]: from='mgr.? 172.21.15.106:0/2350971553' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-02T15:26:20.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:19 smithi106 ceph-mon[25843]: from='mgr.? 172.21.15.106:0/2350971553' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T15:26:20.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:19 smithi087 ceph-mon[24900]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:20.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:19 smithi087 ceph-mon[24900]: Standby manager daemon x started 2024-04-02T15:26:20.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:19 smithi087 ceph-mon[24900]: from='mgr.? 172.21.15.106:0/2350971553' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-02T15:26:20.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:19 smithi087 ceph-mon[24900]: from='mgr.? 172.21.15.106:0/2350971553' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T15:26:20.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:19 smithi087 ceph-mon[24900]: from='mgr.? 172.21.15.106:0/2350971553' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-02T15:26:20.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:19 smithi087 ceph-mon[24900]: from='mgr.? 172.21.15.106:0/2350971553' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T15:26:20.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:19 smithi087 ceph-mon[29707]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:20.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:19 smithi087 ceph-mon[29707]: Standby manager daemon x started 2024-04-02T15:26:20.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:19 smithi087 ceph-mon[29707]: from='mgr.? 172.21.15.106:0/2350971553' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-02T15:26:20.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:19 smithi087 ceph-mon[29707]: from='mgr.? 172.21.15.106:0/2350971553' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T15:26:20.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:19 smithi087 ceph-mon[29707]: from='mgr.? 172.21.15.106:0/2350971553' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-02T15:26:20.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:19 smithi087 ceph-mon[29707]: from='mgr.? 172.21.15.106:0/2350971553' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T15:26:21.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:20 smithi106 ceph-mon[25843]: mgrmap e14: y(active, since 63s), standbys: x 2024-04-02T15:26:21.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:20 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-02T15:26:21.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:20 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/1335905013' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "13d89f7c-4e33-4521-a94a-ff0658e7dccf"}]: dispatch 2024-04-02T15:26:21.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:20 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/1335905013' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "13d89f7c-4e33-4521-a94a-ff0658e7dccf"}]': finished 2024-04-02T15:26:21.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:20 smithi106 ceph-mon[25843]: osdmap e5: 1 total, 0 up, 1 in 2024-04-02T15:26:21.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:20 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T15:26:21.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:20 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/314582362' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T15:26:21.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:20 smithi106 ceph-mon[25843]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:21.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:20 smithi087 ceph-mon[24900]: mgrmap e14: y(active, since 63s), standbys: x 2024-04-02T15:26:21.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:20 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-02T15:26:21.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:20 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/1335905013' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "13d89f7c-4e33-4521-a94a-ff0658e7dccf"}]: dispatch 2024-04-02T15:26:21.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:20 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/1335905013' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "13d89f7c-4e33-4521-a94a-ff0658e7dccf"}]': finished 2024-04-02T15:26:21.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:20 smithi087 ceph-mon[24900]: osdmap e5: 1 total, 0 up, 1 in 2024-04-02T15:26:21.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:20 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T15:26:21.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:20 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/314582362' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T15:26:21.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:20 smithi087 ceph-mon[24900]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:21.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:20 smithi087 ceph-mon[29707]: mgrmap e14: y(active, since 63s), standbys: x 2024-04-02T15:26:21.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:20 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-02T15:26:21.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:20 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/1335905013' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "13d89f7c-4e33-4521-a94a-ff0658e7dccf"}]: dispatch 2024-04-02T15:26:21.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:20 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/1335905013' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "13d89f7c-4e33-4521-a94a-ff0658e7dccf"}]': finished 2024-04-02T15:26:21.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:20 smithi087 ceph-mon[29707]: osdmap e5: 1 total, 0 up, 1 in 2024-04-02T15:26:21.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:20 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T15:26:21.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:20 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/314582362' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T15:26:21.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:20 smithi087 ceph-mon[29707]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:24.043 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:23 smithi087 ceph-mon[24900]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:24.043 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:23 smithi087 ceph-mon[29707]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:24.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:23 smithi106 ceph-mon[25843]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:25.438 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:25 smithi087 ceph-mon[24900]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:25.438 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:25 smithi087 ceph-mon[29707]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:25.615 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:25 smithi106 ceph-mon[25843]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:28.298 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:27 smithi087 ceph-mon[24900]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:28.298 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:27 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-02T15:26:28.298 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:27 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:28.298 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:26:28 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:26:28] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:26:28.298 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:27 smithi087 ceph-mon[29707]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:28.298 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:27 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-02T15:26:28.299 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:27 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:28.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:27 smithi106 ceph-mon[25843]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:28.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:27 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-02T15:26:28.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:27 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:29.176 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:28 smithi087 ceph-mon[24900]: Deploying daemon osd.0 on smithi087 2024-04-02T15:26:29.176 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:28 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:29.176 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:28 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:29.177 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:28 smithi087 ceph-mon[29707]: Deploying daemon osd.0 on smithi087 2024-04-02T15:26:29.177 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:28 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:29.177 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:28 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:29.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:28 smithi106 ceph-mon[25843]: Deploying daemon osd.0 on smithi087 2024-04-02T15:26:29.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:28 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:29.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:28 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:29.921 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:29 smithi087 ceph-mon[24900]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:29.921 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:29 smithi087 ceph-mon[29707]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:30.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:29 smithi106 ceph-mon[25843]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:32.019 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:31 smithi087 ceph-mon[24900]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:32.019 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:31 smithi087 ceph-mon[29707]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:32.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:31 smithi106 ceph-mon[25843]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:33.235 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:32 smithi087 ceph-mon[24900]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:33.235 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:32 smithi087 ceph-mon[29707]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:33.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:32 smithi106 ceph-mon[25843]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:34.491 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:34 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:34.491 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:34 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:26:34.491 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:34 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:34.491 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:34 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:26:34.491 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:34 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:34.511 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:34 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:34.511 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:34 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:26:34.511 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:34 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:34.511 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:34 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:26:34.512 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:34 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:34.512 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:34 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:34.512 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:34 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:26:34.512 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:34 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:34.512 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:34 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:26:34.512 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:34 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:35.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:35 smithi087 ceph-mon[24900]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:35.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:35 smithi087 ceph-mon[29707]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:35.479 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:35 smithi106 ceph-mon[25843]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:35.673 INFO:teuthology.orchestra.run.smithi087.stdout:Created osd(s) 0 on host 'smithi087' 2024-04-02T15:26:35.994 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:26:35 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:26:35] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:26:36.278 DEBUG:teuthology.orchestra.run.smithi087:osd.0> sudo journalctl -f -n 0 -u ceph-0400842c-f105-11ee-b647-cb9ed24678a4@osd.0.service 2024-04-02T15:26:36.280 INFO:tasks.cephadm:Deploying osd.1 on smithi087 with /dev/vg_nvme/lv_3... 2024-04-02T15:26:36.281 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-02T15:26:36.869 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:36 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:36.869 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:36 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:26:36.869 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:36 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:36.869 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:36 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:26:36.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:36 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:26:36.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:36 smithi087 ceph-mon[24900]: Adjusting osd_memory_target on smithi087 to 15040M 2024-04-02T15:26:36.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:36 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:36.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:36 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:36.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:36 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:36.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:36 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:36.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:36 smithi087 ceph-mon[24900]: from='osd.0 [v2:172.21.15.87:6802/2380439245,v1:172.21.15.87:6803/2380439245]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-02T15:26:36.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:36 smithi087 ceph-mon[24900]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-02T15:26:36.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:36 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:36.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:36 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:26:36.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:36 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:36.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:36 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:26:36.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:36 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:26:36.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:36 smithi087 ceph-mon[29707]: Adjusting osd_memory_target on smithi087 to 15040M 2024-04-02T15:26:36.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:36 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:36.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:36 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:36.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:36 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:36.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:36 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:36.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:36 smithi087 ceph-mon[29707]: from='osd.0 [v2:172.21.15.87:6802/2380439245,v1:172.21.15.87:6803/2380439245]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-02T15:26:36.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:36 smithi087 ceph-mon[29707]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-02T15:26:37.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:36 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:37.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:36 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:26:37.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:36 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:37.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:36 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:26:37.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:36 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:26:37.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:36 smithi106 ceph-mon[25843]: Adjusting osd_memory_target on smithi087 to 15040M 2024-04-02T15:26:37.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:36 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:37.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:36 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:37.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:36 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:37.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:36 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:37.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:36 smithi106 ceph-mon[25843]: from='osd.0 [v2:172.21.15.87:6802/2380439245,v1:172.21.15.87:6803/2380439245]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-02T15:26:37.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:36 smithi106 ceph-mon[25843]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-02T15:26:38.119 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:37 smithi087 ceph-mon[29707]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:38.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:37 smithi087 ceph-mon[29707]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-02T15:26:38.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:37 smithi087 ceph-mon[29707]: osdmap e6: 1 total, 0 up, 1 in 2024-04-02T15:26:38.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:37 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T15:26:38.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:37 smithi087 ceph-mon[29707]: from='osd.0 [v2:172.21.15.87:6802/2380439245,v1:172.21.15.87:6803/2380439245]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-02T15:26:38.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:37 smithi087 ceph-mon[29707]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-02T15:26:38.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:37 smithi087 ceph-mon[24900]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:38.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:37 smithi087 ceph-mon[24900]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-02T15:26:38.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:37 smithi087 ceph-mon[24900]: osdmap e6: 1 total, 0 up, 1 in 2024-04-02T15:26:38.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:37 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T15:26:38.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:37 smithi087 ceph-mon[24900]: from='osd.0 [v2:172.21.15.87:6802/2380439245,v1:172.21.15.87:6803/2380439245]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-02T15:26:38.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:37 smithi087 ceph-mon[24900]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-02T15:26:38.121 INFO:journalctl@ceph.osd.0.smithi087.stdout:Apr 02 15:26:37 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-0[33084]: 2024-04-02T15:26:37.903+0000 7f13ab7fd700 -1 osd.0 0 waiting for initial osdmap 2024-04-02T15:26:38.121 INFO:journalctl@ceph.osd.0.smithi087.stdout:Apr 02 15:26:37 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-0[33084]: 2024-04-02T15:26:37.905+0000 7f13a7997700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T15:26:38.165 INFO:teuthology.orchestra.run.smithi087.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-02T15:26:38.165 INFO:teuthology.orchestra.run.smithi087.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-02T15:26:38.165 INFO:teuthology.orchestra.run.smithi087.stderr: stderr: 10+0 records in 2024-04-02T15:26:38.166 INFO:teuthology.orchestra.run.smithi087.stderr:10+0 records out 2024-04-02T15:26:38.166 INFO:teuthology.orchestra.run.smithi087.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0127768 s, 821 MB/s 2024-04-02T15:26:38.166 INFO:teuthology.orchestra.run.smithi087.stderr:--> Zapping successful for: 2024-04-02T15:26:38.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:37 smithi106 ceph-mon[25843]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:38.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:37 smithi106 ceph-mon[25843]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-02T15:26:38.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:37 smithi106 ceph-mon[25843]: osdmap e6: 1 total, 0 up, 1 in 2024-04-02T15:26:38.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:37 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T15:26:38.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:37 smithi106 ceph-mon[25843]: from='osd.0 [v2:172.21.15.87:6802/2380439245,v1:172.21.15.87:6803/2380439245]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-02T15:26:38.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:37 smithi106 ceph-mon[25843]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-02T15:26:38.774 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi087:vg_nvme/lv_3 2024-04-02T15:26:39.101 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:38 smithi087 ceph-mon[24900]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]': finished 2024-04-02T15:26:39.101 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:38 smithi087 ceph-mon[24900]: osdmap e7: 1 total, 0 up, 1 in 2024-04-02T15:26:39.101 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:38 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T15:26:39.101 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:38 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T15:26:39.101 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:38 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T15:26:39.101 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:38 smithi087 ceph-mon[29707]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]': finished 2024-04-02T15:26:39.101 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:38 smithi087 ceph-mon[29707]: osdmap e7: 1 total, 0 up, 1 in 2024-04-02T15:26:39.102 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:38 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T15:26:39.102 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:38 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T15:26:39.102 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:38 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T15:26:39.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:38 smithi106 ceph-mon[25843]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]': finished 2024-04-02T15:26:39.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:38 smithi106 ceph-mon[25843]: osdmap e7: 1 total, 0 up, 1 in 2024-04-02T15:26:39.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:38 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T15:26:39.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:38 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T15:26:39.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:38 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T15:26:40.031 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:39 smithi087 ceph-mon[24900]: purged_snaps scrub starts 2024-04-02T15:26:40.031 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:39 smithi087 ceph-mon[24900]: purged_snaps scrub ok 2024-04-02T15:26:40.031 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:39 smithi087 ceph-mon[24900]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:40.031 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:39 smithi087 ceph-mon[24900]: osd.0 [v2:172.21.15.87:6802/2380439245,v1:172.21.15.87:6803/2380439245] boot 2024-04-02T15:26:40.031 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:39 smithi087 ceph-mon[24900]: osdmap e8: 1 total, 1 up, 1 in 2024-04-02T15:26:40.031 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:39 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T15:26:40.032 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:39 smithi087 ceph-mon[29707]: purged_snaps scrub starts 2024-04-02T15:26:40.032 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:39 smithi087 ceph-mon[29707]: purged_snaps scrub ok 2024-04-02T15:26:40.032 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:39 smithi087 ceph-mon[29707]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:40.032 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:39 smithi087 ceph-mon[29707]: osd.0 [v2:172.21.15.87:6802/2380439245,v1:172.21.15.87:6803/2380439245] boot 2024-04-02T15:26:40.032 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:39 smithi087 ceph-mon[29707]: osdmap e8: 1 total, 1 up, 1 in 2024-04-02T15:26:40.032 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:39 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T15:26:40.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:39 smithi106 ceph-mon[25843]: purged_snaps scrub starts 2024-04-02T15:26:40.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:39 smithi106 ceph-mon[25843]: purged_snaps scrub ok 2024-04-02T15:26:40.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:39 smithi106 ceph-mon[25843]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T15:26:40.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:39 smithi106 ceph-mon[25843]: osd.0 [v2:172.21.15.87:6802/2380439245,v1:172.21.15.87:6803/2380439245] boot 2024-04-02T15:26:40.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:39 smithi106 ceph-mon[25843]: osdmap e8: 1 total, 1 up, 1 in 2024-04-02T15:26:40.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:39 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T15:26:41.304 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:41 smithi087 ceph-mon[24900]: osdmap e9: 1 total, 1 up, 1 in 2024-04-02T15:26:41.304 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:41 smithi087 ceph-mon[24900]: from='client.24145 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi087:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:26:41.304 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:41 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T15:26:41.304 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:41 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T15:26:41.304 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:41 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:41.304 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:41 smithi087 ceph-mon[24900]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T15:26:41.304 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:41 smithi087 ceph-mon[29707]: osdmap e9: 1 total, 1 up, 1 in 2024-04-02T15:26:41.304 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:41 smithi087 ceph-mon[29707]: from='client.24145 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi087:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:26:41.305 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:41 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T15:26:41.305 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:41 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T15:26:41.305 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:41 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:41.305 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:41 smithi087 ceph-mon[29707]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T15:26:41.615 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:41 smithi106 ceph-mon[25843]: osdmap e9: 1 total, 1 up, 1 in 2024-04-02T15:26:41.615 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:41 smithi106 ceph-mon[25843]: from='client.24145 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi087:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:26:41.615 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:41 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T15:26:41.615 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:41 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T15:26:41.615 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:41 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:41.615 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:41 smithi106 ceph-mon[25843]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T15:26:43.903 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:26:43 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:26:43] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:26:44.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:43 smithi106 ceph-mon[25843]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T15:26:44.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:43 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/2428187332' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f8e09079-dd20-411b-a160-5d6906d1d7ed"}]: dispatch 2024-04-02T15:26:44.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:43 smithi106 ceph-mon[25843]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f8e09079-dd20-411b-a160-5d6906d1d7ed"}]: dispatch 2024-04-02T15:26:44.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:43 smithi106 ceph-mon[25843]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f8e09079-dd20-411b-a160-5d6906d1d7ed"}]': finished 2024-04-02T15:26:44.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:43 smithi106 ceph-mon[25843]: osdmap e10: 2 total, 1 up, 2 in 2024-04-02T15:26:44.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:43 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T15:26:44.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:43 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:44.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:43 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:44.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:43 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/3709364322' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T15:26:44.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:43 smithi087 ceph-mon[24900]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T15:26:44.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:43 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/2428187332' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f8e09079-dd20-411b-a160-5d6906d1d7ed"}]: dispatch 2024-04-02T15:26:44.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:43 smithi087 ceph-mon[24900]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f8e09079-dd20-411b-a160-5d6906d1d7ed"}]: dispatch 2024-04-02T15:26:44.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:43 smithi087 ceph-mon[24900]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f8e09079-dd20-411b-a160-5d6906d1d7ed"}]': finished 2024-04-02T15:26:44.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:43 smithi087 ceph-mon[24900]: osdmap e10: 2 total, 1 up, 2 in 2024-04-02T15:26:44.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:43 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T15:26:44.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:43 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:44.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:43 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:44.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:43 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/3709364322' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T15:26:44.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:43 smithi087 ceph-mon[29707]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T15:26:44.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:43 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/2428187332' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f8e09079-dd20-411b-a160-5d6906d1d7ed"}]: dispatch 2024-04-02T15:26:44.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:43 smithi087 ceph-mon[29707]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f8e09079-dd20-411b-a160-5d6906d1d7ed"}]: dispatch 2024-04-02T15:26:44.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:43 smithi087 ceph-mon[29707]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f8e09079-dd20-411b-a160-5d6906d1d7ed"}]': finished 2024-04-02T15:26:44.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:43 smithi087 ceph-mon[29707]: osdmap e10: 2 total, 1 up, 2 in 2024-04-02T15:26:44.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:43 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T15:26:44.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:43 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:44.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:43 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:44.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:43 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/3709364322' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T15:26:45.018 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:44 smithi087 ceph-mon[24900]: Detected new or changed devices on smithi087 2024-04-02T15:26:45.020 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:44 smithi087 ceph-mon[29707]: Detected new or changed devices on smithi087 2024-04-02T15:26:45.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:44 smithi106 ceph-mon[25843]: Detected new or changed devices on smithi087 2024-04-02T15:26:46.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:45 smithi087 ceph-mon[24900]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T15:26:46.119 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:45 smithi087 ceph-mon[29707]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T15:26:46.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:45 smithi106 ceph-mon[25843]: pgmap v44: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T15:26:48.210 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:47 smithi087 ceph-mon[24900]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T15:26:48.210 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:47 smithi087 ceph-mon[29707]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T15:26:48.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:47 smithi106 ceph-mon[25843]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T15:26:49.345 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:48 smithi087 ceph-mon[24900]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T15:26:49.345 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:48 smithi087 ceph-mon[29707]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T15:26:49.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:48 smithi106 ceph-mon[25843]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T15:26:50.485 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:50 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-02T15:26:50.485 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:50 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:50.485 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:50 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-02T15:26:50.486 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:50 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:50.615 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:50 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-02T15:26:50.615 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:50 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:51.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:51 smithi087 ceph-mon[24900]: Deploying daemon osd.1 on smithi087 2024-04-02T15:26:51.369 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:51 smithi087 ceph-mon[24900]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T15:26:51.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:51 smithi087 ceph-mon[29707]: Deploying daemon osd.1 on smithi087 2024-04-02T15:26:51.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:51 smithi087 ceph-mon[29707]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T15:26:51.615 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:51 smithi106 ceph-mon[25843]: Deploying daemon osd.1 on smithi087 2024-04-02T15:26:51.615 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:51 smithi106 ceph-mon[25843]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T15:26:54.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:53 smithi087 ceph-mon[24900]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T15:26:54.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:53 smithi087 ceph-mon[29707]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T15:26:54.318 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:53 smithi106 ceph-mon[25843]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T15:26:55.498 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:55 smithi087 ceph-mon[24900]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T15:26:55.498 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:55 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:55.498 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:55 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:26:55.498 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:55 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:55.499 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:55 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:26:55.499 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:55 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:55.499 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:55 smithi087 ceph-mon[29707]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T15:26:55.499 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:55 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:55.499 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:55 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:26:55.499 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:55 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:55.499 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:55 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:26:55.499 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:55 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:55.515 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:55 smithi106 ceph-mon[25843]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T15:26:55.515 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:55 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:55.515 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:55 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:26:55.515 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:55 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:55.515 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:55 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:26:55.515 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:55 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:56.232 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:26:55 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:26:55] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:26:57.062 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:56 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:57.062 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:56 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:57.062 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:56 smithi106 ceph-mon[25843]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T15:26:57.279 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:56 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:57.279 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:56 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:57.279 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:56 smithi087 ceph-mon[29707]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T15:26:57.279 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:56 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:57.280 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:56 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:57.280 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:56 smithi087 ceph-mon[24900]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T15:26:57.552 INFO:teuthology.orchestra.run.smithi087.stdout:Created osd(s) 1 on host 'smithi087' 2024-04-02T15:26:58.157 DEBUG:teuthology.orchestra.run.smithi087:osd.1> sudo journalctl -f -n 0 -u ceph-0400842c-f105-11ee-b647-cb9ed24678a4@osd.1.service 2024-04-02T15:26:58.159 INFO:tasks.cephadm:Deploying osd.2 on smithi087 with /dev/vg_nvme/lv_2... 2024-04-02T15:26:58.159 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-02T15:26:58.837 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:58 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:58.837 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:58 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:26:58.837 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:58 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:58.838 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:58 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:26:58.838 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:58 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:26:58.838 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:58 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:26:58.838 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:58 smithi087 ceph-mon[29707]: Adjusting osd_memory_target on smithi087 to 7520M 2024-04-02T15:26:58.838 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:58 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:58.838 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:58 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:58.838 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:58 smithi087 ceph-mon[29707]: from='osd.1 [v2:172.21.15.87:6810/1909345406,v1:172.21.15.87:6811/1909345406]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-02T15:26:58.838 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:58 smithi087 ceph-mon[29707]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-02T15:26:58.838 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:58 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:58.838 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:58 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:26:58.838 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:58 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:58.838 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:58 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:26:58.839 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:58 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:26:58.839 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:58 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:26:58.839 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:58 smithi087 ceph-mon[24900]: Adjusting osd_memory_target on smithi087 to 7520M 2024-04-02T15:26:58.839 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:58 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:58.839 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:58 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:58.839 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:58 smithi087 ceph-mon[24900]: from='osd.1 [v2:172.21.15.87:6810/1909345406,v1:172.21.15.87:6811/1909345406]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-02T15:26:58.839 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:58 smithi087 ceph-mon[24900]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-02T15:26:58.865 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:58 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:58.865 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:58 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:26:58.865 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:58 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:26:58.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:58 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:26:58.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:58 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:26:58.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:58 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:26:58.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:58 smithi106 ceph-mon[25843]: Adjusting osd_memory_target on smithi087 to 7520M 2024-04-02T15:26:58.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:58 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:58.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:58 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:26:58.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:58 smithi106 ceph-mon[25843]: from='osd.1 [v2:172.21.15.87:6810/1909345406,v1:172.21.15.87:6811/1909345406]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-02T15:26:58.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:58 smithi106 ceph-mon[25843]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-02T15:26:59.725 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:59 smithi087 ceph-mon[29707]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-02T15:26:59.725 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:59 smithi087 ceph-mon[29707]: osdmap e11: 2 total, 1 up, 2 in 2024-04-02T15:26:59.725 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:59 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T15:26:59.726 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:59 smithi087 ceph-mon[29707]: from='osd.1 [v2:172.21.15.87:6810/1909345406,v1:172.21.15.87:6811/1909345406]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-02T15:26:59.726 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:59 smithi087 ceph-mon[29707]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-02T15:26:59.726 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:26:59 smithi087 ceph-mon[29707]: pgmap v52: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T15:26:59.726 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:59 smithi087 ceph-mon[24900]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-02T15:26:59.726 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:59 smithi087 ceph-mon[24900]: osdmap e11: 2 total, 1 up, 2 in 2024-04-02T15:26:59.726 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:59 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T15:26:59.726 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:59 smithi087 ceph-mon[24900]: from='osd.1 [v2:172.21.15.87:6810/1909345406,v1:172.21.15.87:6811/1909345406]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-02T15:26:59.726 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:59 smithi087 ceph-mon[24900]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-02T15:26:59.726 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:26:59 smithi087 ceph-mon[24900]: pgmap v52: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T15:26:59.727 INFO:journalctl@ceph.osd.1.smithi087.stdout:Apr 02 15:26:59 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-1[35962]: 2024-04-02T15:26:59.592+0000 7fa439a41700 -1 osd.1 0 waiting for initial osdmap 2024-04-02T15:26:59.727 INFO:journalctl@ceph.osd.1.smithi087.stdout:Apr 02 15:26:59 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-1[35962]: 2024-04-02T15:26:59.595+0000 7fa4343d8700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T15:26:59.865 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:59 smithi106 ceph-mon[25843]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-02T15:26:59.865 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:59 smithi106 ceph-mon[25843]: osdmap e11: 2 total, 1 up, 2 in 2024-04-02T15:26:59.865 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:59 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T15:26:59.865 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:59 smithi106 ceph-mon[25843]: from='osd.1 [v2:172.21.15.87:6810/1909345406,v1:172.21.15.87:6811/1909345406]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-02T15:26:59.865 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:59 smithi106 ceph-mon[25843]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-02T15:26:59.865 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:26:59 smithi106 ceph-mon[25843]: pgmap v52: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-02T15:27:00.188 INFO:teuthology.orchestra.run.smithi087.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-02T15:27:00.188 INFO:teuthology.orchestra.run.smithi087.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-02T15:27:00.188 INFO:teuthology.orchestra.run.smithi087.stderr: stderr: 10+0 records in 2024-04-02T15:27:00.188 INFO:teuthology.orchestra.run.smithi087.stderr:10+0 records out 2024-04-02T15:27:00.188 INFO:teuthology.orchestra.run.smithi087.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0126236 s, 831 MB/s 2024-04-02T15:27:00.188 INFO:teuthology.orchestra.run.smithi087.stderr:--> Zapping successful for: 2024-04-02T15:27:00.788 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi087:vg_nvme/lv_2 2024-04-02T15:27:00.814 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:00 smithi087 ceph-mon[24900]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]': finished 2024-04-02T15:27:00.814 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:00 smithi087 ceph-mon[24900]: osdmap e12: 2 total, 1 up, 2 in 2024-04-02T15:27:00.814 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:00 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T15:27:00.814 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:00 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T15:27:00.814 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:00 smithi087 ceph-mon[29707]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]': finished 2024-04-02T15:27:00.814 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:00 smithi087 ceph-mon[29707]: osdmap e12: 2 total, 1 up, 2 in 2024-04-02T15:27:00.814 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:00 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T15:27:00.815 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:00 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T15:27:00.865 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:00 smithi106 ceph-mon[25843]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]': finished 2024-04-02T15:27:00.865 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:00 smithi106 ceph-mon[25843]: osdmap e12: 2 total, 1 up, 2 in 2024-04-02T15:27:00.865 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:00 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T15:27:00.865 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:00 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T15:27:01.865 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:01 smithi106 ceph-mon[25843]: purged_snaps scrub starts 2024-04-02T15:27:01.865 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:01 smithi106 ceph-mon[25843]: purged_snaps scrub ok 2024-04-02T15:27:01.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:01 smithi106 ceph-mon[25843]: osd.1 [v2:172.21.15.87:6810/1909345406,v1:172.21.15.87:6811/1909345406] boot 2024-04-02T15:27:01.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:01 smithi106 ceph-mon[25843]: osdmap e13: 2 total, 2 up, 2 in 2024-04-02T15:27:01.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:01 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T15:27:01.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:01 smithi106 ceph-mon[25843]: pgmap v55: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-02T15:27:01.869 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:01 smithi087 ceph-mon[24900]: purged_snaps scrub starts 2024-04-02T15:27:01.869 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:01 smithi087 ceph-mon[24900]: purged_snaps scrub ok 2024-04-02T15:27:01.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:01 smithi087 ceph-mon[24900]: osd.1 [v2:172.21.15.87:6810/1909345406,v1:172.21.15.87:6811/1909345406] boot 2024-04-02T15:27:01.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:01 smithi087 ceph-mon[24900]: osdmap e13: 2 total, 2 up, 2 in 2024-04-02T15:27:01.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:01 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T15:27:01.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:01 smithi087 ceph-mon[24900]: pgmap v55: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-02T15:27:01.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:01 smithi087 ceph-mon[29707]: purged_snaps scrub starts 2024-04-02T15:27:01.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:01 smithi087 ceph-mon[29707]: purged_snaps scrub ok 2024-04-02T15:27:01.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:01 smithi087 ceph-mon[29707]: osd.1 [v2:172.21.15.87:6810/1909345406,v1:172.21.15.87:6811/1909345406] boot 2024-04-02T15:27:01.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:01 smithi087 ceph-mon[29707]: osdmap e13: 2 total, 2 up, 2 in 2024-04-02T15:27:01.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:01 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T15:27:01.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:01 smithi087 ceph-mon[29707]: pgmap v55: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-04-02T15:27:02.865 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:02 smithi106 ceph-mon[25843]: osdmap e14: 2 total, 2 up, 2 in 2024-04-02T15:27:02.865 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:02 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T15:27:02.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:02 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T15:27:02.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:02 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:02.869 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:02 smithi087 ceph-mon[24900]: osdmap e14: 2 total, 2 up, 2 in 2024-04-02T15:27:02.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:02 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T15:27:02.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:02 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T15:27:02.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:02 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:02.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:02 smithi087 ceph-mon[29707]: osdmap e14: 2 total, 2 up, 2 in 2024-04-02T15:27:02.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:02 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T15:27:02.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:02 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T15:27:02.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:02 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:03.684 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:03 smithi087 ceph-mon[29707]: from='client.24172 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi087:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:27:03.684 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:03 smithi087 ceph-mon[29707]: pgmap v57: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-02T15:27:03.684 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:03 smithi087 ceph-mon[24900]: from='client.24172 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi087:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:27:03.684 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:03 smithi087 ceph-mon[24900]: pgmap v57: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-02T15:27:03.865 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:03 smithi106 ceph-mon[25843]: from='client.24172 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi087:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:27:03.865 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:03 smithi106 ceph-mon[25843]: pgmap v57: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-02T15:27:04.119 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:27:03 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:27:03] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:27:04.773 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:04 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:04.773 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:04 smithi087 ceph-mon[24900]: Detected new or changed devices on smithi087 2024-04-02T15:27:04.773 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:04 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:04.774 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:04 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:27:04.774 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:04 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:04.774 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:04 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:27:04.774 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:04 smithi087 ceph-mon[24900]: pgmap v58: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-02T15:27:04.774 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:04 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:04.774 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:04 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:04.774 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:04 smithi087 ceph-mon[29707]: Detected new or changed devices on smithi087 2024-04-02T15:27:04.774 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:04 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:04.774 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:04 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:27:04.774 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:04 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:04.774 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:04 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:27:04.774 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:04 smithi087 ceph-mon[29707]: pgmap v58: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-02T15:27:04.775 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:04 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:05.010 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:04 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:05.010 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:04 smithi106 ceph-mon[25843]: Detected new or changed devices on smithi087 2024-04-02T15:27:05.010 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:04 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:05.010 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:04 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:27:05.011 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:04 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:05.011 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:04 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:27:05.011 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:04 smithi106 ceph-mon[25843]: pgmap v58: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-02T15:27:05.011 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:04 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:05.865 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:05 smithi106 ceph-mon[25843]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-04-02T15:27:05.865 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:05 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/3610720266' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "55868ded-a2a8-48c7-8fb2-d452831755d6"}]: dispatch 2024-04-02T15:27:05.865 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:05 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/3610720266' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "55868ded-a2a8-48c7-8fb2-d452831755d6"}]': finished 2024-04-02T15:27:05.865 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:05 smithi106 ceph-mon[25843]: osdmap e15: 3 total, 2 up, 3 in 2024-04-02T15:27:05.865 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:05 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T15:27:06.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:05 smithi087 ceph-mon[29707]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-04-02T15:27:06.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:05 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/3610720266' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "55868ded-a2a8-48c7-8fb2-d452831755d6"}]: dispatch 2024-04-02T15:27:06.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:05 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/3610720266' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "55868ded-a2a8-48c7-8fb2-d452831755d6"}]': finished 2024-04-02T15:27:06.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:05 smithi087 ceph-mon[29707]: osdmap e15: 3 total, 2 up, 3 in 2024-04-02T15:27:06.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:05 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T15:27:06.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:05 smithi087 ceph-mon[24900]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-04-02T15:27:06.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:05 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/3610720266' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "55868ded-a2a8-48c7-8fb2-d452831755d6"}]: dispatch 2024-04-02T15:27:06.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:05 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/3610720266' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "55868ded-a2a8-48c7-8fb2-d452831755d6"}]': finished 2024-04-02T15:27:06.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:05 smithi087 ceph-mon[24900]: osdmap e15: 3 total, 2 up, 3 in 2024-04-02T15:27:06.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:05 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T15:27:06.620 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:27:06 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:27:06] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:27:07.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:06 smithi106 ceph-mon[25843]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T15:27:07.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:06 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:07.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:06 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:07.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:06 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:27:07.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:06 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:07.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:06 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:27:07.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:06 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:07.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:06 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/2043275868' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T15:27:07.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:06 smithi087 ceph-mon[24900]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T15:27:07.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:06 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:07.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:06 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:07.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:06 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:27:07.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:06 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:07.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:06 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:27:07.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:06 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:07.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:06 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/2043275868' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T15:27:07.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:06 smithi087 ceph-mon[29707]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T15:27:07.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:06 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:07.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:06 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:07.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:06 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:27:07.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:06 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:07.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:06 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:27:07.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:06 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:07.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:06 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/2043275868' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T15:27:08.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:07 smithi106 ceph-mon[25843]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T15:27:08.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:07 smithi106 ceph-mon[25843]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-04-02T15:27:08.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:07 smithi106 ceph-mon[25843]: Cluster is now healthy 2024-04-02T15:27:08.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:07 smithi087 ceph-mon[24900]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T15:27:08.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:07 smithi087 ceph-mon[24900]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-04-02T15:27:08.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:07 smithi087 ceph-mon[24900]: Cluster is now healthy 2024-04-02T15:27:08.119 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:07 smithi087 ceph-mon[29707]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T15:27:08.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:07 smithi087 ceph-mon[29707]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-04-02T15:27:08.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:07 smithi087 ceph-mon[29707]: Cluster is now healthy 2024-04-02T15:27:10.002 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:09 smithi087 ceph-mon[29707]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T15:27:10.002 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:09 smithi087 ceph-mon[24900]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T15:27:10.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:09 smithi106 ceph-mon[25843]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T15:27:11.838 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:11 smithi087 ceph-mon[24900]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T15:27:11.839 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:11 smithi087 ceph-mon[29707]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T15:27:12.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:11 smithi106 ceph-mon[25843]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T15:27:14.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:13 smithi106 ceph-mon[25843]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T15:27:14.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:13 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-02T15:27:14.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:13 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:14.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:13 smithi087 ceph-mon[24900]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T15:27:14.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:13 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-02T15:27:14.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:13 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:14.119 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:13 smithi087 ceph-mon[29707]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T15:27:14.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:13 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-02T15:27:14.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:13 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:14.945 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:14 smithi087 ceph-mon[29707]: Deploying daemon osd.2 on smithi087 2024-04-02T15:27:14.945 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:14 smithi087 ceph-mon[24900]: Deploying daemon osd.2 on smithi087 2024-04-02T15:27:15.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:14 smithi106 ceph-mon[25843]: Deploying daemon osd.2 on smithi087 2024-04-02T15:27:15.918 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:15 smithi087 ceph-mon[29707]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T15:27:15.919 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:15 smithi087 ceph-mon[24900]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T15:27:16.077 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:15 smithi106 ceph-mon[25843]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T15:27:16.261 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:27:16 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:27:16] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:27:17.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:17 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:17.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:17 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:17.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:17 smithi106 ceph-mon[25843]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T15:27:17.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:17 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:27:17.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:17 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:27:17.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:17 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:17.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:17 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:17.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:17 smithi087 ceph-mon[24900]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T15:27:17.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:17 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:27:17.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:17 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:27:17.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:17 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:17.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:17 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:17.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:17 smithi087 ceph-mon[29707]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T15:27:17.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:17 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:27:17.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:17 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:27:19.268 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:18 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:19.268 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:18 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:27:19.268 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:18 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:19.268 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:18 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:27:19.268 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:18 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:19.268 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:18 smithi087 ceph-mon[24900]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T15:27:19.268 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:18 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:19.269 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:18 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:27:19.269 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:18 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:19.269 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:18 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:27:19.269 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:18 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:19.269 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:18 smithi087 ceph-mon[29707]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T15:27:19.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:18 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:19.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:18 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:27:19.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:18 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:19.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:18 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:27:19.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:18 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:19.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:18 smithi106 ceph-mon[25843]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T15:27:20.493 INFO:teuthology.orchestra.run.smithi087.stdout:Created osd(s) 2 on host 'smithi087' 2024-04-02T15:27:21.163 DEBUG:teuthology.orchestra.run.smithi087:osd.2> sudo journalctl -f -n 0 -u ceph-0400842c-f105-11ee-b647-cb9ed24678a4@osd.2.service 2024-04-02T15:27:21.207 INFO:tasks.cephadm:Deploying osd.3 on smithi087 with /dev/vg_nvme/lv_1... 2024-04-02T15:27:21.207 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-02T15:27:21.751 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:21 smithi087 ceph-mon[24900]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T15:27:21.751 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:21 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:21.751 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:21 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:27:21.751 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:21 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:21.752 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:21 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:27:21.752 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:21 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:27:21.752 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:21 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:27:21.752 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:21 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:27:21.752 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:21 smithi087 ceph-mon[24900]: Adjusting osd_memory_target on smithi087 to 5013M 2024-04-02T15:27:21.752 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:21 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:21.752 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:21 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:21.752 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:21 smithi087 ceph-mon[24900]: from='osd.2 [v2:172.21.15.87:6818/2886422817,v1:172.21.15.87:6819/2886422817]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-02T15:27:21.752 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:21 smithi087 ceph-mon[24900]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-02T15:27:21.752 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:21 smithi087 ceph-mon[29707]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T15:27:21.752 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:21 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:21.752 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:21 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:27:21.752 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:21 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:21.753 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:21 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:27:21.753 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:21 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:27:21.753 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:21 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:27:21.753 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:21 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:27:21.753 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:21 smithi087 ceph-mon[29707]: Adjusting osd_memory_target on smithi087 to 5013M 2024-04-02T15:27:21.753 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:21 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:21.753 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:21 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:21.753 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:21 smithi087 ceph-mon[29707]: from='osd.2 [v2:172.21.15.87:6818/2886422817,v1:172.21.15.87:6819/2886422817]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-02T15:27:21.753 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:21 smithi087 ceph-mon[29707]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-02T15:27:21.865 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:21 smithi106 ceph-mon[25843]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T15:27:21.865 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:21 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:21.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:21 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:27:21.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:21 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:21.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:21 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:27:21.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:21 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:27:21.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:21 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:27:21.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:21 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:27:21.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:21 smithi106 ceph-mon[25843]: Adjusting osd_memory_target on smithi087 to 5013M 2024-04-02T15:27:21.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:21 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:21.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:21 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:21.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:21 smithi106 ceph-mon[25843]: from='osd.2 [v2:172.21.15.87:6818/2886422817,v1:172.21.15.87:6819/2886422817]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-02T15:27:21.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:21 smithi106 ceph-mon[25843]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-02T15:27:22.766 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:22 smithi087 ceph-mon[29707]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-02T15:27:22.766 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:22 smithi087 ceph-mon[29707]: osdmap e16: 3 total, 2 up, 3 in 2024-04-02T15:27:22.767 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:22 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T15:27:22.767 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:22 smithi087 ceph-mon[29707]: from='osd.2 [v2:172.21.15.87:6818/2886422817,v1:172.21.15.87:6819/2886422817]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-02T15:27:22.767 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:22 smithi087 ceph-mon[29707]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-02T15:27:22.767 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:22 smithi087 ceph-mon[24900]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-02T15:27:22.767 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:22 smithi087 ceph-mon[24900]: osdmap e16: 3 total, 2 up, 3 in 2024-04-02T15:27:22.767 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:22 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T15:27:22.767 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:22 smithi087 ceph-mon[24900]: from='osd.2 [v2:172.21.15.87:6818/2886422817,v1:172.21.15.87:6819/2886422817]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-02T15:27:22.767 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:22 smithi087 ceph-mon[24900]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-02T15:27:22.767 INFO:journalctl@ceph.osd.2.smithi087.stdout:Apr 02 15:27:22 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-2[38901]: 2024-04-02T15:27:22.538+0000 7fa5417e5700 -1 osd.2 0 waiting for initial osdmap 2024-04-02T15:27:22.768 INFO:journalctl@ceph.osd.2.smithi087.stdout:Apr 02 15:27:22 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-2[38901]: 2024-04-02T15:27:22.542+0000 7fa53e180700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T15:27:22.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:22 smithi106 ceph-mon[25843]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-02T15:27:22.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:22 smithi106 ceph-mon[25843]: osdmap e16: 3 total, 2 up, 3 in 2024-04-02T15:27:22.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:22 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T15:27:22.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:22 smithi106 ceph-mon[25843]: from='osd.2 [v2:172.21.15.87:6818/2886422817,v1:172.21.15.87:6819/2886422817]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-02T15:27:22.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:22 smithi106 ceph-mon[25843]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-02T15:27:23.238 INFO:teuthology.orchestra.run.smithi087.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-02T15:27:23.238 INFO:teuthology.orchestra.run.smithi087.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-02T15:27:23.238 INFO:teuthology.orchestra.run.smithi087.stderr: stderr: 10+0 records in 2024-04-02T15:27:23.238 INFO:teuthology.orchestra.run.smithi087.stderr:10+0 records out 2024-04-02T15:27:23.238 INFO:teuthology.orchestra.run.smithi087.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0125781 s, 834 MB/s 2024-04-02T15:27:23.238 INFO:teuthology.orchestra.run.smithi087.stderr:--> Zapping successful for: 2024-04-02T15:27:23.811 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:23 smithi087 ceph-mon[29707]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T15:27:23.811 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:23 smithi087 ceph-mon[29707]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]': finished 2024-04-02T15:27:23.811 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:23 smithi087 ceph-mon[29707]: osdmap e17: 3 total, 2 up, 3 in 2024-04-02T15:27:23.811 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:23 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T15:27:23.811 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:23 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T15:27:23.812 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:23 smithi087 ceph-mon[24900]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T15:27:23.812 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:23 smithi087 ceph-mon[24900]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]': finished 2024-04-02T15:27:23.812 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:23 smithi087 ceph-mon[24900]: osdmap e17: 3 total, 2 up, 3 in 2024-04-02T15:27:23.812 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:23 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T15:27:23.812 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:23 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T15:27:23.837 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi087:vg_nvme/lv_1 2024-04-02T15:27:23.865 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:23 smithi106 ceph-mon[25843]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T15:27:23.865 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:23 smithi106 ceph-mon[25843]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]': finished 2024-04-02T15:27:23.865 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:23 smithi106 ceph-mon[25843]: osdmap e17: 3 total, 2 up, 3 in 2024-04-02T15:27:23.865 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:23 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T15:27:23.865 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:23 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T15:27:24.772 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:24 smithi087 ceph-mon[24900]: purged_snaps scrub starts 2024-04-02T15:27:24.773 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:24 smithi087 ceph-mon[24900]: purged_snaps scrub ok 2024-04-02T15:27:24.773 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:24 smithi087 ceph-mon[24900]: osd.2 [v2:172.21.15.87:6818/2886422817,v1:172.21.15.87:6819/2886422817] boot 2024-04-02T15:27:24.773 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:24 smithi087 ceph-mon[24900]: osdmap e18: 3 total, 3 up, 3 in 2024-04-02T15:27:24.773 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:24 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T15:27:24.773 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:24 smithi087 ceph-mon[29707]: purged_snaps scrub starts 2024-04-02T15:27:24.773 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:24 smithi087 ceph-mon[29707]: purged_snaps scrub ok 2024-04-02T15:27:24.773 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:24 smithi087 ceph-mon[29707]: osd.2 [v2:172.21.15.87:6818/2886422817,v1:172.21.15.87:6819/2886422817] boot 2024-04-02T15:27:24.773 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:24 smithi087 ceph-mon[29707]: osdmap e18: 3 total, 3 up, 3 in 2024-04-02T15:27:24.773 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:24 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T15:27:24.865 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:24 smithi106 ceph-mon[25843]: purged_snaps scrub starts 2024-04-02T15:27:24.865 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:24 smithi106 ceph-mon[25843]: purged_snaps scrub ok 2024-04-02T15:27:24.865 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:24 smithi106 ceph-mon[25843]: osd.2 [v2:172.21.15.87:6818/2886422817,v1:172.21.15.87:6819/2886422817] boot 2024-04-02T15:27:24.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:24 smithi106 ceph-mon[25843]: osdmap e18: 3 total, 3 up, 3 in 2024-04-02T15:27:24.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:24 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T15:27:25.755 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:25 smithi087 ceph-mon[24900]: pgmap v73: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T15:27:25.755 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:25 smithi087 ceph-mon[24900]: osdmap e19: 3 total, 3 up, 3 in 2024-04-02T15:27:25.755 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:25 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-02T15:27:25.756 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:25 smithi087 ceph-mon[29707]: pgmap v73: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T15:27:25.756 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:25 smithi087 ceph-mon[29707]: osdmap e19: 3 total, 3 up, 3 in 2024-04-02T15:27:25.756 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:25 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-02T15:27:25.865 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:25 smithi106 ceph-mon[25843]: pgmap v73: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-02T15:27:25.865 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:25 smithi106 ceph-mon[25843]: osdmap e19: 3 total, 3 up, 3 in 2024-04-02T15:27:25.865 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:25 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-02T15:27:26.458 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:27:26 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:27:26] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:27:26.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:26 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-02T15:27:26.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:26 smithi087 ceph-mon[29707]: osdmap e20: 3 total, 3 up, 3 in 2024-04-02T15:27:26.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:26 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-02T15:27:26.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:26 smithi087 ceph-mon[29707]: from='client.24199 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi087:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:27:26.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:26 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T15:27:26.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:26 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T15:27:26.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:26 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:26.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:26 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:26.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:26 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:26.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:26 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:27:26.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:26 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:26.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:26 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:27:26.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:26 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:26.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:26 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-02T15:27:26.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:26 smithi087 ceph-mon[24900]: osdmap e20: 3 total, 3 up, 3 in 2024-04-02T15:27:26.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:26 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-02T15:27:26.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:26 smithi087 ceph-mon[24900]: from='client.24199 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi087:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:27:26.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:26 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T15:27:26.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:26 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T15:27:26.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:26 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:26.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:26 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:26.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:26 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:26.872 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:26 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:27:26.872 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:26 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:26.872 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:26 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:27:26.872 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:26 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:27.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:26 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-02T15:27:27.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:26 smithi106 ceph-mon[25843]: osdmap e20: 3 total, 3 up, 3 in 2024-04-02T15:27:27.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:26 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-02T15:27:27.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:26 smithi106 ceph-mon[25843]: from='client.24199 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi087:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:27:27.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:26 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T15:27:27.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:26 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T15:27:27.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:26 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:27.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:26 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:27.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:26 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:27.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:26 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:27:27.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:26 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:27.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:26 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:27:27.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:26 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:27.457 INFO:journalctl@ceph.osd.0.smithi087.stdout:Apr 02 15:27:27 smithi087 sudo[40160]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-02T15:27:27.457 INFO:journalctl@ceph.osd.0.smithi087.stdout:Apr 02 15:27:27 smithi087 sudo[40160]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T15:27:27.457 INFO:journalctl@ceph.osd.0.smithi087.stdout:Apr 02 15:27:27 smithi087 sudo[40160]: pam_unix(sudo:session): session closed for user root 2024-04-02T15:27:27.777 INFO:journalctl@ceph.osd.0.smithi087.stdout:Apr 02 15:27:27 smithi087 sudo[40182]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-02T15:27:27.777 INFO:journalctl@ceph.osd.0.smithi087.stdout:Apr 02 15:27:27 smithi087 sudo[40182]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T15:27:27.777 INFO:journalctl@ceph.osd.0.smithi087.stdout:Apr 02 15:27:27 smithi087 sudo[40182]: pam_unix(sudo:session): session closed for user root 2024-04-02T15:27:27.778 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:27 smithi087 ceph-mon[24900]: pgmap v76: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-02T15:27:27.778 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:27 smithi087 ceph-mon[24900]: Detected new or changed devices on smithi087 2024-04-02T15:27:27.778 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:27 smithi087 ceph-mon[24900]: pgmap v77: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-02T15:27:27.778 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:27 smithi087 ceph-mon[24900]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-04-02T15:27:27.778 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:27 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-02T15:27:27.778 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:27 smithi087 ceph-mon[24900]: osdmap e21: 3 total, 3 up, 3 in 2024-04-02T15:27:27.778 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:27 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:27.778 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:27 smithi087 ceph-mon[29707]: pgmap v76: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-02T15:27:27.778 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:27 smithi087 ceph-mon[29707]: Detected new or changed devices on smithi087 2024-04-02T15:27:27.778 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:27 smithi087 ceph-mon[29707]: pgmap v77: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-02T15:27:27.779 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:27 smithi087 ceph-mon[29707]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-04-02T15:27:27.779 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:27 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-02T15:27:27.779 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:27 smithi087 ceph-mon[29707]: osdmap e21: 3 total, 3 up, 3 in 2024-04-02T15:27:27.779 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:27 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:28.035 INFO:journalctl@ceph.osd.1.smithi087.stdout:Apr 02 15:27:27 smithi087 sudo[40204]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-02T15:27:28.035 INFO:journalctl@ceph.osd.1.smithi087.stdout:Apr 02 15:27:27 smithi087 sudo[40204]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T15:27:28.035 INFO:journalctl@ceph.osd.1.smithi087.stdout:Apr 02 15:27:27 smithi087 sudo[40204]: pam_unix(sudo:session): session closed for user root 2024-04-02T15:27:28.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:27 smithi106 ceph-mon[25843]: pgmap v76: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-02T15:27:28.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:27 smithi106 ceph-mon[25843]: Detected new or changed devices on smithi087 2024-04-02T15:27:28.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:27 smithi106 ceph-mon[25843]: pgmap v77: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-02T15:27:28.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:27 smithi106 ceph-mon[25843]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-04-02T15:27:28.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:27 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-02T15:27:28.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:27 smithi106 ceph-mon[25843]: osdmap e21: 3 total, 3 up, 3 in 2024-04-02T15:27:28.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:27 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:28.286 INFO:journalctl@ceph.osd.1.smithi087.stdout:Apr 02 15:27:28 smithi087 sudo[40221]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-02T15:27:28.286 INFO:journalctl@ceph.osd.1.smithi087.stdout:Apr 02 15:27:28 smithi087 sudo[40221]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T15:27:28.286 INFO:journalctl@ceph.osd.1.smithi087.stdout:Apr 02 15:27:28 smithi087 sudo[40221]: pam_unix(sudo:session): session closed for user root 2024-04-02T15:27:28.549 INFO:journalctl@ceph.osd.2.smithi087.stdout:Apr 02 15:27:28 smithi087 sudo[40262]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-02T15:27:28.550 INFO:journalctl@ceph.osd.2.smithi087.stdout:Apr 02 15:27:28 smithi087 sudo[40262]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T15:27:28.550 INFO:journalctl@ceph.osd.2.smithi087.stdout:Apr 02 15:27:28 smithi087 sudo[40262]: pam_unix(sudo:session): session closed for user root 2024-04-02T15:27:28.802 INFO:journalctl@ceph.osd.2.smithi087.stdout:Apr 02 15:27:28 smithi087 sudo[40292]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-02T15:27:28.802 INFO:journalctl@ceph.osd.2.smithi087.stdout:Apr 02 15:27:28 smithi087 sudo[40292]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T15:27:28.802 INFO:journalctl@ceph.osd.2.smithi087.stdout:Apr 02 15:27:28 smithi087 sudo[40292]: pam_unix(sudo:session): session closed for user root 2024-04-02T15:27:29.062 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:28 smithi087 sudo[40323]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-02T15:27:29.062 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:28 smithi087 sudo[40323]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T15:27:29.062 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:28 smithi087 sudo[40323]: pam_unix(sudo:session): session closed for user root 2024-04-02T15:27:29.321 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:29 smithi087 ceph-mon[29707]: osdmap e22: 3 total, 3 up, 3 in 2024-04-02T15:27:29.393 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:29 smithi087 sudo[40351]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-02T15:27:29.393 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:29 smithi087 sudo[40351]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T15:27:29.393 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:29 smithi087 sudo[40351]: pam_unix(sudo:session): session closed for user root 2024-04-02T15:27:29.394 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:29 smithi087 ceph-mon[24900]: osdmap e22: 3 total, 3 up, 3 in 2024-04-02T15:27:29.395 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:29 smithi106 ceph-mon[25843]: osdmap e22: 3 total, 3 up, 3 in 2024-04-02T15:27:29.578 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:29 smithi087 sudo[40360]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-02T15:27:29.578 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:29 smithi087 sudo[40360]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T15:27:29.578 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:29 smithi087 sudo[40360]: pam_unix(sudo:session): session closed for user root 2024-04-02T15:27:29.872 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:29 smithi087 sudo[40391]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-02T15:27:29.873 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:29 smithi087 sudo[40391]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T15:27:29.874 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:29 smithi087 sudo[40391]: pam_unix(sudo:session): session closed for user root 2024-04-02T15:27:30.075 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:29 smithi106 sudo[28355]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-02T15:27:30.075 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:29 smithi106 sudo[28355]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T15:27:30.075 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:29 smithi106 sudo[28355]: pam_unix(sudo:session): session closed for user root 2024-04-02T15:27:30.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:30 smithi106 sudo[28358]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-02T15:27:30.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:30 smithi106 sudo[28358]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-02T15:27:30.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:30 smithi106 sudo[28358]: pam_unix(sudo:session): session closed for user root 2024-04-02T15:27:30.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:30 smithi106 ceph-mon[25843]: pgmap v80: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-02T15:27:30.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:30 smithi106 ceph-mon[25843]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-02T15:27:30.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:30 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/1926132708' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fba679e6-6a92-4943-bbc9-3c446311dfc5"}]: dispatch 2024-04-02T15:27:30.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:30 smithi106 ceph-mon[25843]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-02T15:27:30.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:30 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T15:27:30.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:30 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:27:30.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:30 smithi106 ceph-mon[25843]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fba679e6-6a92-4943-bbc9-3c446311dfc5"}]: dispatch 2024-04-02T15:27:30.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:30 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:27:30.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:30 smithi106 ceph-mon[25843]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fba679e6-6a92-4943-bbc9-3c446311dfc5"}]': finished 2024-04-02T15:27:30.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:30 smithi106 ceph-mon[25843]: osdmap e23: 4 total, 3 up, 4 in 2024-04-02T15:27:30.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:30 smithi106 ceph-mon[25843]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-02T15:27:30.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:30 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T15:27:30.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:30 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T15:27:30.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:30 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:27:30.368 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:30 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:27:30.368 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:30 smithi106 ceph-mon[25843]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-02T15:27:30.368 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:30 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:30.368 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:30 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T15:27:30.368 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:30 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:27:30.368 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:30 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:27:30.368 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:30 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:27:30.368 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:30 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:30.368 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:30 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:27:30.368 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:30 smithi106 ceph-mon[25843]: pgmap v82: 1 pgs: 1 creating+peering; 0 B data, 16 MiB used, 268 GiB / 268 GiB avail 2024-04-02T15:27:30.368 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:30 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/2843112351' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T15:27:30.369 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:27:30] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:27:30.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[29707]: pgmap v80: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-02T15:27:30.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[29707]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-02T15:27:30.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/1926132708' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fba679e6-6a92-4943-bbc9-3c446311dfc5"}]: dispatch 2024-04-02T15:27:30.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[29707]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-02T15:27:30.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T15:27:30.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:27:30.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[29707]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fba679e6-6a92-4943-bbc9-3c446311dfc5"}]: dispatch 2024-04-02T15:27:30.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:27:30.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[29707]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fba679e6-6a92-4943-bbc9-3c446311dfc5"}]': finished 2024-04-02T15:27:30.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[29707]: osdmap e23: 4 total, 3 up, 4 in 2024-04-02T15:27:30.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[29707]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-02T15:27:30.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T15:27:30.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T15:27:30.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:27:30.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:27:30.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[29707]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-02T15:27:30.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:30.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T15:27:30.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:27:30.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:27:30.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:27:30.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:30.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:27:30.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[29707]: pgmap v82: 1 pgs: 1 creating+peering; 0 B data, 16 MiB used, 268 GiB / 268 GiB avail 2024-04-02T15:27:30.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/2843112351' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T15:27:30.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[24900]: pgmap v80: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-02T15:27:30.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[24900]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-02T15:27:30.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/1926132708' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fba679e6-6a92-4943-bbc9-3c446311dfc5"}]: dispatch 2024-04-02T15:27:30.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[24900]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-02T15:27:30.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T15:27:30.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:27:30.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[24900]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fba679e6-6a92-4943-bbc9-3c446311dfc5"}]: dispatch 2024-04-02T15:27:30.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:27:30.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[24900]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fba679e6-6a92-4943-bbc9-3c446311dfc5"}]': finished 2024-04-02T15:27:30.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[24900]: osdmap e23: 4 total, 3 up, 4 in 2024-04-02T15:27:30.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[24900]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-02T15:27:30.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T15:27:30.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T15:27:30.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:27:30.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:27:30.373 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[24900]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-02T15:27:30.373 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:30.373 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T15:27:30.373 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:27:30.373 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:27:30.373 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:27:30.373 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:30.373 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:27:30.373 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[24900]: pgmap v82: 1 pgs: 1 creating+peering; 0 B data, 16 MiB used, 268 GiB / 268 GiB avail 2024-04-02T15:27:30.373 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:30 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/2843112351' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T15:27:31.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:31 smithi106 ceph-mon[25843]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-02T15:27:31.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:31 smithi106 ceph-mon[25843]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-02T15:27:31.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:31 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:31.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:31 smithi106 ceph-mon[25843]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-02T15:27:31.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:31 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:31.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:31 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:31.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:31 smithi106 ceph-mon[25843]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-04-02T15:27:31.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:31 smithi106 ceph-mon[25843]: Cluster is now healthy 2024-04-02T15:27:31.369 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:31 smithi087 ceph-mon[29707]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-02T15:27:31.369 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:31 smithi087 ceph-mon[29707]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-02T15:27:31.369 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:31 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:31.369 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:31 smithi087 ceph-mon[29707]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-02T15:27:31.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:31 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:31.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:31 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:31.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:31 smithi087 ceph-mon[29707]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-04-02T15:27:31.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:31 smithi087 ceph-mon[29707]: Cluster is now healthy 2024-04-02T15:27:31.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:31 smithi087 ceph-mon[24900]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-02T15:27:31.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:31 smithi087 ceph-mon[24900]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-02T15:27:31.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:31 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:31.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:31 smithi087 ceph-mon[24900]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-02T15:27:31.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:31 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:31.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:31 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:31.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:31 smithi087 ceph-mon[24900]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-04-02T15:27:31.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:31 smithi087 ceph-mon[24900]: Cluster is now healthy 2024-04-02T15:27:32.237 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:32 smithi087 ceph-mon[29707]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T15:27:32.238 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:32 smithi087 ceph-mon[24900]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T15:27:32.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:32 smithi106 ceph-mon[25843]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T15:27:34.907 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:34 smithi087 ceph-mon[29707]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T15:27:34.907 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:34 smithi087 ceph-mon[24900]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T15:27:35.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:34 smithi106 ceph-mon[25843]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T15:27:36.595 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:27:36 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:27:36] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:27:36.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:36 smithi087 ceph-mon[24900]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T15:27:36.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:36 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:36.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:36 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:36.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:36 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-02T15:27:36.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:36 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:36.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:36 smithi087 ceph-mon[29707]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T15:27:36.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:36 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:36.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:36 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:36.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:36 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-02T15:27:36.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:36 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:36.980 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:36 smithi106 ceph-mon[25843]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T15:27:36.980 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:36 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:36.981 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:36 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:36.981 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:36 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-02T15:27:36.981 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:36 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:37.750 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:37 smithi087 ceph-mon[29707]: Deploying daemon osd.3 on smithi087 2024-04-02T15:27:37.750 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:37 smithi087 ceph-mon[24900]: Deploying daemon osd.3 on smithi087 2024-04-02T15:27:37.804 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:37 smithi106 ceph-mon[25843]: Deploying daemon osd.3 on smithi087 2024-04-02T15:27:38.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:38 smithi087 ceph-mon[24900]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T15:27:38.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:38 smithi087 ceph-mon[29707]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T15:27:39.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:38 smithi106 ceph-mon[25843]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T15:27:40.520 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:40 smithi087 ceph-mon[29707]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T15:27:40.520 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:40 smithi087 ceph-mon[24900]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T15:27:40.615 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:40 smithi106 ceph-mon[25843]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T15:27:42.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:42 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:42.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:42 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:27:42.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:42 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:42.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:42 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:27:42.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:42 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:42.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:42 smithi087 ceph-mon[24900]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T15:27:42.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:42 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:42.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:42 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:27:42.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:42 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:42.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:42 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:27:42.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:42 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:42.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:42 smithi087 ceph-mon[29707]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T15:27:42.615 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:42 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:42.615 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:42 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:27:42.615 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:42 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:42.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:42 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:27:42.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:42 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:42.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:42 smithi106 ceph-mon[25843]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T15:27:43.874 INFO:teuthology.orchestra.run.smithi087.stdout:Created osd(s) 3 on host 'smithi087' 2024-04-02T15:27:44.650 DEBUG:teuthology.orchestra.run.smithi087:osd.3> sudo journalctl -f -n 0 -u ceph-0400842c-f105-11ee-b647-cb9ed24678a4@osd.3.service 2024-04-02T15:27:44.653 INFO:tasks.cephadm:Deploying osd.4 on smithi106 with /dev/vg_nvme/lv_4... 2024-04-02T15:27:44.653 DEBUG:teuthology.orchestra.run.smithi106:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-02T15:27:44.971 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:44 smithi106 ceph-mon[25843]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T15:27:44.972 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:44 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:44.972 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:44 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:27:44.972 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:44 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:44.972 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:44 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:27:44.972 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:44 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:27:44.972 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:44 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:27:44.972 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:44 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:27:44.972 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:44 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:27:44.972 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:44 smithi106 ceph-mon[25843]: Adjusting osd_memory_target on smithi087 to 3760M 2024-04-02T15:27:44.972 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:44 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:44.972 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:44 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:44.972 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:44 smithi106 ceph-mon[25843]: from='osd.3 [v2:172.21.15.87:6826/1277310151,v1:172.21.15.87:6827/1277310151]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-02T15:27:45.122 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:44 smithi087 ceph-mon[24900]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T15:27:45.122 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:44 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:45.122 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:44 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:27:45.122 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:44 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:45.122 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:44 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:27:45.122 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:44 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:27:45.122 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:44 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:27:45.122 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:44 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:27:45.122 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:44 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:27:45.123 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:44 smithi087 ceph-mon[24900]: Adjusting osd_memory_target on smithi087 to 3760M 2024-04-02T15:27:45.123 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:44 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:45.123 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:44 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:45.123 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:44 smithi087 ceph-mon[24900]: from='osd.3 [v2:172.21.15.87:6826/1277310151,v1:172.21.15.87:6827/1277310151]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-02T15:27:45.123 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:44 smithi087 ceph-mon[29707]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T15:27:45.123 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:44 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:45.123 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:44 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:27:45.123 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:44 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:45.123 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:44 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:27:45.123 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:44 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:27:45.123 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:44 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:27:45.123 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:44 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:27:45.123 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:44 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:27:45.124 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:44 smithi087 ceph-mon[29707]: Adjusting osd_memory_target on smithi087 to 3760M 2024-04-02T15:27:45.124 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:44 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:45.124 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:44 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:45.124 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:44 smithi087 ceph-mon[29707]: from='osd.3 [v2:172.21.15.87:6826/1277310151,v1:172.21.15.87:6827/1277310151]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-02T15:27:46.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:46 smithi106 ceph-mon[25843]: from='osd.3 [v2:172.21.15.87:6826/1277310151,v1:172.21.15.87:6827/1277310151]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-02T15:27:46.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:46 smithi106 ceph-mon[25843]: osdmap e24: 4 total, 3 up, 4 in 2024-04-02T15:27:46.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:46 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T15:27:46.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:46 smithi106 ceph-mon[25843]: from='osd.3 [v2:172.21.15.87:6826/1277310151,v1:172.21.15.87:6827/1277310151]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-02T15:27:46.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:46 smithi106 ceph-mon[25843]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T15:27:46.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:46 smithi087 ceph-mon[24900]: from='osd.3 [v2:172.21.15.87:6826/1277310151,v1:172.21.15.87:6827/1277310151]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-02T15:27:46.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:46 smithi087 ceph-mon[24900]: osdmap e24: 4 total, 3 up, 4 in 2024-04-02T15:27:46.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:46 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T15:27:46.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:46 smithi087 ceph-mon[24900]: from='osd.3 [v2:172.21.15.87:6826/1277310151,v1:172.21.15.87:6827/1277310151]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-02T15:27:46.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:46 smithi087 ceph-mon[24900]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T15:27:46.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:46 smithi087 ceph-mon[29707]: from='osd.3 [v2:172.21.15.87:6826/1277310151,v1:172.21.15.87:6827/1277310151]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-02T15:27:46.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:46 smithi087 ceph-mon[29707]: osdmap e24: 4 total, 3 up, 4 in 2024-04-02T15:27:46.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:46 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T15:27:46.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:46 smithi087 ceph-mon[29707]: from='osd.3 [v2:172.21.15.87:6826/1277310151,v1:172.21.15.87:6827/1277310151]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]: dispatch 2024-04-02T15:27:46.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:46 smithi087 ceph-mon[29707]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T15:27:46.371 INFO:journalctl@ceph.osd.3.smithi087.stdout:Apr 02 15:27:46 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-3[41873]: 2024-04-02T15:27:46.189+0000 7f0a72f4c700 -1 osd.3 0 waiting for initial osdmap 2024-04-02T15:27:46.371 INFO:journalctl@ceph.osd.3.smithi087.stdout:Apr 02 15:27:46 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-3[41873]: 2024-04-02T15:27:46.193+0000 7f0a6d8e3700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T15:27:46.472 INFO:teuthology.orchestra.run.smithi106.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-02T15:27:46.472 INFO:teuthology.orchestra.run.smithi106.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-02T15:27:46.472 INFO:teuthology.orchestra.run.smithi106.stderr: stderr: 10+0 records in 2024-04-02T15:27:46.472 INFO:teuthology.orchestra.run.smithi106.stderr:10+0 records out 2024-04-02T15:27:46.473 INFO:teuthology.orchestra.run.smithi106.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0113937 s, 920 MB/s 2024-04-02T15:27:46.473 INFO:teuthology.orchestra.run.smithi106.stderr:--> Zapping successful for: 2024-04-02T15:27:47.043 DEBUG:teuthology.orchestra.run.smithi106:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi106:vg_nvme/lv_4 2024-04-02T15:27:47.348 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:47 smithi106 ceph-mon[25843]: from='osd.3 [v2:172.21.15.87:6826/1277310151,v1:172.21.15.87:6827/1277310151]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]': finished 2024-04-02T15:27:47.349 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:47 smithi106 ceph-mon[25843]: osdmap e25: 4 total, 3 up, 4 in 2024-04-02T15:27:47.349 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:47 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T15:27:47.349 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:47 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T15:27:47.619 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:27:47 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:27:47] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:27:47.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:47 smithi087 ceph-mon[24900]: from='osd.3 [v2:172.21.15.87:6826/1277310151,v1:172.21.15.87:6827/1277310151]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]': finished 2024-04-02T15:27:47.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:47 smithi087 ceph-mon[24900]: osdmap e25: 4 total, 3 up, 4 in 2024-04-02T15:27:47.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:47 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T15:27:47.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:47 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T15:27:47.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:47 smithi087 ceph-mon[29707]: from='osd.3 [v2:172.21.15.87:6826/1277310151,v1:172.21.15.87:6827/1277310151]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi087", "root=default"]}]': finished 2024-04-02T15:27:47.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:47 smithi087 ceph-mon[29707]: osdmap e25: 4 total, 3 up, 4 in 2024-04-02T15:27:47.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:47 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T15:27:47.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:47 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T15:27:48.317 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:48 smithi106 ceph-mon[25843]: purged_snaps scrub starts 2024-04-02T15:27:48.317 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:48 smithi106 ceph-mon[25843]: purged_snaps scrub ok 2024-04-02T15:27:48.317 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:48 smithi106 ceph-mon[25843]: osd.3 [v2:172.21.15.87:6826/1277310151,v1:172.21.15.87:6827/1277310151] boot 2024-04-02T15:27:48.317 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:48 smithi106 ceph-mon[25843]: osdmap e26: 4 total, 4 up, 4 in 2024-04-02T15:27:48.317 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:48 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T15:27:48.317 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:48 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:48.317 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:48 smithi106 ceph-mon[25843]: Detected new or changed devices on smithi087 2024-04-02T15:27:48.317 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:48 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:48.317 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:48 smithi106 ceph-mon[25843]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T15:27:48.446 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:48 smithi087 ceph-mon[24900]: purged_snaps scrub starts 2024-04-02T15:27:48.446 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:48 smithi087 ceph-mon[24900]: purged_snaps scrub ok 2024-04-02T15:27:48.446 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:48 smithi087 ceph-mon[24900]: osd.3 [v2:172.21.15.87:6826/1277310151,v1:172.21.15.87:6827/1277310151] boot 2024-04-02T15:27:48.446 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:48 smithi087 ceph-mon[24900]: osdmap e26: 4 total, 4 up, 4 in 2024-04-02T15:27:48.446 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:48 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T15:27:48.446 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:48 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:48.446 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:48 smithi087 ceph-mon[24900]: Detected new or changed devices on smithi087 2024-04-02T15:27:48.446 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:48 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:48.447 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:48 smithi087 ceph-mon[24900]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T15:27:48.447 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:48 smithi087 ceph-mon[29707]: purged_snaps scrub starts 2024-04-02T15:27:48.447 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:48 smithi087 ceph-mon[29707]: purged_snaps scrub ok 2024-04-02T15:27:48.447 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:48 smithi087 ceph-mon[29707]: osd.3 [v2:172.21.15.87:6826/1277310151,v1:172.21.15.87:6827/1277310151] boot 2024-04-02T15:27:48.447 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:48 smithi087 ceph-mon[29707]: osdmap e26: 4 total, 4 up, 4 in 2024-04-02T15:27:48.447 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:48 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T15:27:48.447 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:48 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:48.447 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:48 smithi087 ceph-mon[29707]: Detected new or changed devices on smithi087 2024-04-02T15:27:48.447 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:48 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:48.447 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:48 smithi087 ceph-mon[29707]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-02T15:27:49.587 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:49 smithi087 ceph-mon[24900]: osdmap e27: 4 total, 4 up, 4 in 2024-04-02T15:27:49.587 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:49 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T15:27:49.587 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:49 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T15:27:49.587 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:49 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:49.587 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:49 smithi087 ceph-mon[29707]: osdmap e27: 4 total, 4 up, 4 in 2024-04-02T15:27:49.588 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:49 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T15:27:49.588 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:49 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T15:27:49.588 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:49 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:49.615 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:49 smithi106 ceph-mon[25843]: osdmap e27: 4 total, 4 up, 4 in 2024-04-02T15:27:49.615 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:49 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T15:27:49.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:49 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T15:27:49.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:49 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:50.500 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:50 smithi106 ceph-mon[25843]: from='client.24215 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi106:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:27:50.501 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:50 smithi106 ceph-mon[25843]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T15:27:50.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:50 smithi087 ceph-mon[24900]: from='client.24215 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi106:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:27:50.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:50 smithi087 ceph-mon[24900]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T15:27:50.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:50 smithi087 ceph-mon[29707]: from='client.24215 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi106:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:27:50.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:50 smithi087 ceph-mon[29707]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T15:27:51.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:51 smithi106 ceph-mon[25843]: from='client.? 172.21.15.106:0/562937399' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dcd6edd9-74b5-4ba9-9282-e9325c914835"}]: dispatch 2024-04-02T15:27:51.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:51 smithi106 ceph-mon[25843]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dcd6edd9-74b5-4ba9-9282-e9325c914835"}]: dispatch 2024-04-02T15:27:51.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:51 smithi106 ceph-mon[25843]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "dcd6edd9-74b5-4ba9-9282-e9325c914835"}]': finished 2024-04-02T15:27:51.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:51 smithi106 ceph-mon[25843]: osdmap e28: 5 total, 4 up, 5 in 2024-04-02T15:27:51.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:51 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T15:27:51.619 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:51 smithi087 ceph-mon[24900]: from='client.? 172.21.15.106:0/562937399' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dcd6edd9-74b5-4ba9-9282-e9325c914835"}]: dispatch 2024-04-02T15:27:51.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:51 smithi087 ceph-mon[24900]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dcd6edd9-74b5-4ba9-9282-e9325c914835"}]: dispatch 2024-04-02T15:27:51.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:51 smithi087 ceph-mon[24900]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "dcd6edd9-74b5-4ba9-9282-e9325c914835"}]': finished 2024-04-02T15:27:51.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:51 smithi087 ceph-mon[24900]: osdmap e28: 5 total, 4 up, 5 in 2024-04-02T15:27:51.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:51 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T15:27:51.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:51 smithi087 ceph-mon[29707]: from='client.? 172.21.15.106:0/562937399' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dcd6edd9-74b5-4ba9-9282-e9325c914835"}]: dispatch 2024-04-02T15:27:51.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:51 smithi087 ceph-mon[29707]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "dcd6edd9-74b5-4ba9-9282-e9325c914835"}]: dispatch 2024-04-02T15:27:51.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:51 smithi087 ceph-mon[29707]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "dcd6edd9-74b5-4ba9-9282-e9325c914835"}]': finished 2024-04-02T15:27:51.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:51 smithi087 ceph-mon[29707]: osdmap e28: 5 total, 4 up, 5 in 2024-04-02T15:27:51.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:51 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T15:27:52.615 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:52 smithi106 ceph-mon[25843]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T15:27:52.615 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:52 smithi106 ceph-mon[25843]: from='client.? 172.21.15.106:0/1410097502' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T15:27:52.619 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:52 smithi087 ceph-mon[29707]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T15:27:52.619 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:52 smithi087 ceph-mon[29707]: from='client.? 172.21.15.106:0/1410097502' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T15:27:52.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:52 smithi087 ceph-mon[24900]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T15:27:52.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:52 smithi087 ceph-mon[24900]: from='client.? 172.21.15.106:0/1410097502' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T15:27:55.009 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:54 smithi106 ceph-mon[25843]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T15:27:55.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:54 smithi087 ceph-mon[24900]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T15:27:55.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:54 smithi087 ceph-mon[29707]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T15:27:56.619 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:27:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:27:56] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:27:56.842 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:56 smithi106 ceph-mon[25843]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T15:27:56.842 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:56 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:56.842 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:56 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:57.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:56 smithi087 ceph-mon[29707]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T15:27:57.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:56 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:57.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:56 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:57.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:56 smithi087 ceph-mon[24900]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T15:27:57.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:56 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:57.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:56 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:27:57.865 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:57 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-02T15:27:57.865 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:57 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:58.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:57 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-02T15:27:58.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:57 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:58.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:57 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-02T15:27:58.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:57 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:27:59.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:58 smithi106 ceph-mon[25843]: Deploying daemon osd.4 on smithi106 2024-04-02T15:27:59.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:27:58 smithi106 ceph-mon[25843]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T15:27:59.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:58 smithi087 ceph-mon[24900]: Deploying daemon osd.4 on smithi106 2024-04-02T15:27:59.119 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:27:58 smithi087 ceph-mon[24900]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T15:27:59.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:58 smithi087 ceph-mon[29707]: Deploying daemon osd.4 on smithi106 2024-04-02T15:27:59.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:27:58 smithi087 ceph-mon[29707]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T15:28:00.504 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:00 smithi106 ceph-mon[25843]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T15:28:00.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:00 smithi087 ceph-mon[24900]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T15:28:00.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:00 smithi087 ceph-mon[29707]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T15:28:02.857 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:02 smithi106 ceph-mon[25843]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T15:28:03.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:02 smithi087 ceph-mon[24900]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T15:28:03.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:02 smithi087 ceph-mon[29707]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T15:28:04.619 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:28:04 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:28:04] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:28:05.067 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:04 smithi106 ceph-mon[25843]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T15:28:05.067 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:04 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:05.067 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:04 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:28:05.067 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:04 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:05.068 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:04 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:28:05.068 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:04 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:05.068 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:04 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:05.068 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:04 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:05.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:04 smithi087 ceph-mon[29707]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T15:28:05.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:04 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:05.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:04 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:28:05.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:04 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:05.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:04 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:28:05.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:04 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:05.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:04 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:05.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:04 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:05.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:04 smithi087 ceph-mon[24900]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T15:28:05.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:04 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:05.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:04 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:28:05.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:04 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:05.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:04 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:28:05.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:04 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:05.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:04 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:05.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:04 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:06.697 INFO:teuthology.orchestra.run.smithi106.stdout:Created osd(s) 4 on host 'smithi106' 2024-04-02T15:28:06.993 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:06 smithi106 ceph-mon[25843]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T15:28:06.993 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:06 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:07.096 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:06 smithi087 ceph-mon[29707]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T15:28:07.096 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:06 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:07.096 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:06 smithi087 ceph-mon[24900]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T15:28:07.096 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:06 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:07.258 DEBUG:teuthology.orchestra.run.smithi106:osd.4> sudo journalctl -f -n 0 -u ceph-0400842c-f105-11ee-b647-cb9ed24678a4@osd.4.service 2024-04-02T15:28:07.261 INFO:tasks.cephadm:Deploying osd.5 on smithi106 with /dev/vg_nvme/lv_3... 2024-04-02T15:28:07.261 DEBUG:teuthology.orchestra.run.smithi106:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-02T15:28:07.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:07 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:28:07.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:07 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:07.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:07 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:28:07.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:07 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:28:07.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:07 smithi106 ceph-mon[25843]: Adjusting osd_memory_target on smithi106 to 16064M 2024-04-02T15:28:07.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:07 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:07.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:07 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:07.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:07 smithi106 ceph-mon[25843]: from='osd.4 [v2:172.21.15.106:6800/2668368842,v1:172.21.15.106:6801/2668368842]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-02T15:28:07.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:07 smithi106 ceph-mon[25843]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-02T15:28:08.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:07 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:28:08.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:07 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:08.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:07 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:28:08.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:07 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:28:08.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:07 smithi087 ceph-mon[29707]: Adjusting osd_memory_target on smithi106 to 16064M 2024-04-02T15:28:08.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:07 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:08.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:07 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:08.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:07 smithi087 ceph-mon[29707]: from='osd.4 [v2:172.21.15.106:6800/2668368842,v1:172.21.15.106:6801/2668368842]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-02T15:28:08.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:07 smithi087 ceph-mon[29707]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-02T15:28:08.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:07 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:28:08.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:07 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:08.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:07 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:28:08.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:07 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:28:08.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:07 smithi087 ceph-mon[24900]: Adjusting osd_memory_target on smithi106 to 16064M 2024-04-02T15:28:08.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:07 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:08.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:07 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:08.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:07 smithi087 ceph-mon[24900]: from='osd.4 [v2:172.21.15.106:6800/2668368842,v1:172.21.15.106:6801/2668368842]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-02T15:28:08.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:07 smithi087 ceph-mon[24900]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-02T15:28:08.835 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:08 smithi106 ceph-mon[25843]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T15:28:08.836 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:08 smithi106 ceph-mon[25843]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-02T15:28:08.836 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:08 smithi106 ceph-mon[25843]: osdmap e29: 5 total, 4 up, 5 in 2024-04-02T15:28:08.836 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:08 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T15:28:08.836 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:08 smithi106 ceph-mon[25843]: from='osd.4 [v2:172.21.15.106:6800/2668368842,v1:172.21.15.106:6801/2668368842]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi106", "root=default"]}]: dispatch 2024-04-02T15:28:08.836 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:08 smithi106 ceph-mon[25843]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi106", "root=default"]}]: dispatch 2024-04-02T15:28:08.836 INFO:journalctl@ceph.osd.4.smithi106.stdout:Apr 02 15:28:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-4[30558]: 2024-04-02T15:28:08.767+0000 7fb65bac4700 -1 osd.4 0 waiting for initial osdmap 2024-04-02T15:28:08.836 INFO:journalctl@ceph.osd.4.smithi106.stdout:Apr 02 15:28:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-4[30558]: 2024-04-02T15:28:08.771+0000 7fb65845f700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T15:28:09.096 INFO:teuthology.orchestra.run.smithi106.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-02T15:28:09.096 INFO:teuthology.orchestra.run.smithi106.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-02T15:28:09.096 INFO:teuthology.orchestra.run.smithi106.stderr: stderr: 10+0 records in 2024-04-02T15:28:09.096 INFO:teuthology.orchestra.run.smithi106.stderr:10+0 records out 2024-04-02T15:28:09.096 INFO:teuthology.orchestra.run.smithi106.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0129214 s, 812 MB/s 2024-04-02T15:28:09.096 INFO:teuthology.orchestra.run.smithi106.stderr:--> Zapping successful for: 2024-04-02T15:28:09.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:08 smithi087 ceph-mon[24900]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T15:28:09.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:08 smithi087 ceph-mon[24900]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-02T15:28:09.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:08 smithi087 ceph-mon[24900]: osdmap e29: 5 total, 4 up, 5 in 2024-04-02T15:28:09.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:08 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T15:28:09.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:08 smithi087 ceph-mon[24900]: from='osd.4 [v2:172.21.15.106:6800/2668368842,v1:172.21.15.106:6801/2668368842]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi106", "root=default"]}]: dispatch 2024-04-02T15:28:09.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:08 smithi087 ceph-mon[24900]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi106", "root=default"]}]: dispatch 2024-04-02T15:28:09.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:08 smithi087 ceph-mon[29707]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T15:28:09.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:08 smithi087 ceph-mon[29707]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-02T15:28:09.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:08 smithi087 ceph-mon[29707]: osdmap e29: 5 total, 4 up, 5 in 2024-04-02T15:28:09.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:08 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T15:28:09.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:08 smithi087 ceph-mon[29707]: from='osd.4 [v2:172.21.15.106:6800/2668368842,v1:172.21.15.106:6801/2668368842]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi106", "root=default"]}]: dispatch 2024-04-02T15:28:09.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:08 smithi087 ceph-mon[29707]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi106", "root=default"]}]: dispatch 2024-04-02T15:28:09.679 DEBUG:teuthology.orchestra.run.smithi106:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi106:vg_nvme/lv_3 2024-04-02T15:28:09.973 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:09 smithi106 ceph-mon[25843]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi106", "root=default"]}]': finished 2024-04-02T15:28:09.973 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:09 smithi106 ceph-mon[25843]: osdmap e30: 5 total, 4 up, 5 in 2024-04-02T15:28:09.973 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:09 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T15:28:09.973 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:09 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T15:28:09.973 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:09 smithi106 ceph-mon[25843]: osd.4 [v2:172.21.15.106:6800/2668368842,v1:172.21.15.106:6801/2668368842] boot 2024-04-02T15:28:09.973 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:09 smithi106 ceph-mon[25843]: osdmap e31: 5 total, 5 up, 5 in 2024-04-02T15:28:09.974 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:09 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T15:28:10.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:09 smithi087 ceph-mon[29707]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi106", "root=default"]}]': finished 2024-04-02T15:28:10.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:09 smithi087 ceph-mon[29707]: osdmap e30: 5 total, 4 up, 5 in 2024-04-02T15:28:10.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:09 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T15:28:10.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:09 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T15:28:10.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:09 smithi087 ceph-mon[29707]: osd.4 [v2:172.21.15.106:6800/2668368842,v1:172.21.15.106:6801/2668368842] boot 2024-04-02T15:28:10.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:09 smithi087 ceph-mon[29707]: osdmap e31: 5 total, 5 up, 5 in 2024-04-02T15:28:10.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:09 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T15:28:10.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:09 smithi087 ceph-mon[24900]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi106", "root=default"]}]': finished 2024-04-02T15:28:10.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:09 smithi087 ceph-mon[24900]: osdmap e30: 5 total, 4 up, 5 in 2024-04-02T15:28:10.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:09 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T15:28:10.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:09 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T15:28:10.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:09 smithi087 ceph-mon[24900]: osd.4 [v2:172.21.15.106:6800/2668368842,v1:172.21.15.106:6801/2668368842] boot 2024-04-02T15:28:10.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:09 smithi087 ceph-mon[24900]: osdmap e31: 5 total, 5 up, 5 in 2024-04-02T15:28:10.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:09 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T15:28:11.111 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:10 smithi106 ceph-mon[25843]: purged_snaps scrub starts 2024-04-02T15:28:11.111 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:10 smithi106 ceph-mon[25843]: purged_snaps scrub ok 2024-04-02T15:28:11.111 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:10 smithi106 ceph-mon[25843]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T15:28:11.111 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:10 smithi106 ceph-mon[25843]: osdmap e32: 5 total, 5 up, 5 in 2024-04-02T15:28:11.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:10 smithi087 ceph-mon[24900]: purged_snaps scrub starts 2024-04-02T15:28:11.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:10 smithi087 ceph-mon[24900]: purged_snaps scrub ok 2024-04-02T15:28:11.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:10 smithi087 ceph-mon[24900]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T15:28:11.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:10 smithi087 ceph-mon[24900]: osdmap e32: 5 total, 5 up, 5 in 2024-04-02T15:28:11.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:10 smithi087 ceph-mon[29707]: purged_snaps scrub starts 2024-04-02T15:28:11.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:10 smithi087 ceph-mon[29707]: purged_snaps scrub ok 2024-04-02T15:28:11.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:10 smithi087 ceph-mon[29707]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-02T15:28:11.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:10 smithi087 ceph-mon[29707]: osdmap e32: 5 total, 5 up, 5 in 2024-04-02T15:28:12.615 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:12 smithi106 ceph-mon[25843]: osdmap e33: 5 total, 5 up, 5 in 2024-04-02T15:28:12.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:12 smithi106 ceph-mon[25843]: from='client.24242 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi106:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:28:12.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:12 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T15:28:12.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:12 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T15:28:12.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:12 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:12.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:12 smithi106 ceph-mon[25843]: pgmap v113: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 133 KiB/s, 0 objects/s recovering 2024-04-02T15:28:12.619 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:12 smithi087 ceph-mon[24900]: osdmap e33: 5 total, 5 up, 5 in 2024-04-02T15:28:12.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:12 smithi087 ceph-mon[24900]: from='client.24242 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi106:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:28:12.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:12 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T15:28:12.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:12 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T15:28:12.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:12 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:12.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:12 smithi087 ceph-mon[24900]: pgmap v113: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 133 KiB/s, 0 objects/s recovering 2024-04-02T15:28:12.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:12 smithi087 ceph-mon[29707]: osdmap e33: 5 total, 5 up, 5 in 2024-04-02T15:28:12.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:12 smithi087 ceph-mon[29707]: from='client.24242 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi106:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:28:12.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:12 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T15:28:12.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:12 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T15:28:12.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:12 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:12.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:12 smithi087 ceph-mon[29707]: pgmap v113: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 133 KiB/s, 0 objects/s recovering 2024-04-02T15:28:13.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:13 smithi106 ceph-mon[25843]: osdmap e34: 5 total, 5 up, 5 in 2024-04-02T15:28:13.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:13 smithi087 ceph-mon[24900]: osdmap e34: 5 total, 5 up, 5 in 2024-04-02T15:28:13.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:13 smithi087 ceph-mon[29707]: osdmap e34: 5 total, 5 up, 5 in 2024-04-02T15:28:14.615 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:14 smithi106 ceph-mon[25843]: osdmap e35: 5 total, 5 up, 5 in 2024-04-02T15:28:14.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:14 smithi106 ceph-mon[25843]: pgmap v116: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-04-02T15:28:14.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:14 smithi106 ceph-mon[25843]: from='client.? 172.21.15.106:0/2362136954' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4c205155-70f2-4d96-aebb-3d0e96ba5a95"}]: dispatch 2024-04-02T15:28:14.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:14 smithi106 ceph-mon[25843]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4c205155-70f2-4d96-aebb-3d0e96ba5a95"}]: dispatch 2024-04-02T15:28:14.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:14 smithi106 ceph-mon[25843]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4c205155-70f2-4d96-aebb-3d0e96ba5a95"}]': finished 2024-04-02T15:28:14.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:14 smithi106 ceph-mon[25843]: osdmap e36: 6 total, 5 up, 6 in 2024-04-02T15:28:14.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:14 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T15:28:14.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:14 smithi087 ceph-mon[29707]: osdmap e35: 5 total, 5 up, 5 in 2024-04-02T15:28:14.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:14 smithi087 ceph-mon[29707]: pgmap v116: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-04-02T15:28:14.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:14 smithi087 ceph-mon[29707]: from='client.? 172.21.15.106:0/2362136954' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4c205155-70f2-4d96-aebb-3d0e96ba5a95"}]: dispatch 2024-04-02T15:28:14.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:14 smithi087 ceph-mon[29707]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4c205155-70f2-4d96-aebb-3d0e96ba5a95"}]: dispatch 2024-04-02T15:28:14.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:14 smithi087 ceph-mon[29707]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4c205155-70f2-4d96-aebb-3d0e96ba5a95"}]': finished 2024-04-02T15:28:14.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:14 smithi087 ceph-mon[29707]: osdmap e36: 6 total, 5 up, 6 in 2024-04-02T15:28:14.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:14 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T15:28:14.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:14 smithi087 ceph-mon[24900]: osdmap e35: 5 total, 5 up, 5 in 2024-04-02T15:28:14.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:14 smithi087 ceph-mon[24900]: pgmap v116: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-04-02T15:28:14.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:14 smithi087 ceph-mon[24900]: from='client.? 172.21.15.106:0/2362136954' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4c205155-70f2-4d96-aebb-3d0e96ba5a95"}]: dispatch 2024-04-02T15:28:14.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:14 smithi087 ceph-mon[24900]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4c205155-70f2-4d96-aebb-3d0e96ba5a95"}]: dispatch 2024-04-02T15:28:14.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:14 smithi087 ceph-mon[24900]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4c205155-70f2-4d96-aebb-3d0e96ba5a95"}]': finished 2024-04-02T15:28:14.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:14 smithi087 ceph-mon[24900]: osdmap e36: 6 total, 5 up, 6 in 2024-04-02T15:28:14.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:14 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T15:28:15.506 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:15 smithi087 ceph-mon[24900]: from='client.? 172.21.15.106:0/2983200443' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T15:28:15.506 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:15 smithi087 ceph-mon[29707]: from='client.? 172.21.15.106:0/2983200443' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T15:28:15.506 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:15 smithi106 ceph-mon[25843]: from='client.? 172.21.15.106:0/2983200443' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T15:28:15.870 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:28:15 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:28:15] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:28:16.809 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:16 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:16.809 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:16 smithi106 ceph-mon[25843]: Detected new or changed devices on smithi106 2024-04-02T15:28:16.809 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:16 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:16.809 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:16 smithi106 ceph-mon[25843]: pgmap v118: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 116 KiB/s, 0 objects/s recovering 2024-04-02T15:28:16.852 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:16 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:16.852 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:16 smithi087 ceph-mon[29707]: Detected new or changed devices on smithi106 2024-04-02T15:28:16.852 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:16 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:16.852 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:16 smithi087 ceph-mon[29707]: pgmap v118: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 116 KiB/s, 0 objects/s recovering 2024-04-02T15:28:16.852 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:16 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:16.853 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:16 smithi087 ceph-mon[24900]: Detected new or changed devices on smithi106 2024-04-02T15:28:16.853 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:16 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:16.853 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:16 smithi087 ceph-mon[24900]: pgmap v118: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 116 KiB/s, 0 objects/s recovering 2024-04-02T15:28:17.120 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:28:16 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:28:16.855+0000 7fd8e536d700 -1 calc_pg_upmaps abort due to max <= 0 2024-04-02T15:28:17.837 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:17 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:28:17.837 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:17 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:28:17.869 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:17 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:28:17.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:17 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:28:17.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:17 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:28:17.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:17 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:28:18.615 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:18 smithi106 ceph-mon[25843]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T15:28:18.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:18 smithi087 ceph-mon[24900]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T15:28:18.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:18 smithi087 ceph-mon[29707]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T15:28:20.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:20 smithi106 ceph-mon[25843]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T15:28:20.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:20 smithi087 ceph-mon[24900]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T15:28:20.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:20 smithi087 ceph-mon[29707]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T15:28:21.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:21 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-02T15:28:21.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:21 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:21.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:21 smithi106 ceph-mon[25843]: Deploying daemon osd.5 on smithi106 2024-04-02T15:28:21.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:21 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-02T15:28:21.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:21 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:21.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:21 smithi087 ceph-mon[24900]: Deploying daemon osd.5 on smithi106 2024-04-02T15:28:21.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:21 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-02T15:28:21.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:21 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:21.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:21 smithi087 ceph-mon[29707]: Deploying daemon osd.5 on smithi106 2024-04-02T15:28:22.327 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:22 smithi106 ceph-mon[25843]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T15:28:22.521 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:22 smithi087 ceph-mon[29707]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T15:28:22.521 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:22 smithi087 ceph-mon[24900]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T15:28:24.870 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:28:24 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:28:24] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:28:24.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:24 smithi087 ceph-mon[29707]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T15:28:24.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:24 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:24.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:24 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:24.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:24 smithi087 ceph-mon[24900]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T15:28:24.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:24 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:24.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:24 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:24.888 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:24 smithi106 ceph-mon[25843]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T15:28:24.888 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:24 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:24.888 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:24 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:26.916 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:26 smithi106 ceph-mon[25843]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T15:28:27.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:26 smithi087 ceph-mon[24900]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T15:28:27.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:26 smithi087 ceph-mon[29707]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T15:28:28.281 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:27 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:28.281 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:27 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:28:28.281 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:27 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:28.281 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:27 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:28:28.281 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:27 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:28.282 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:27 smithi106 ceph-mon[25843]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T15:28:28.344 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:27 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:28.344 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:27 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:28:28.344 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:27 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:28.344 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:27 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:28:28.344 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:27 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:28.344 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:27 smithi087 ceph-mon[24900]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T15:28:28.344 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:27 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:28.344 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:27 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:28:28.344 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:27 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:28.344 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:27 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:28:28.344 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:27 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:28.344 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:27 smithi087 ceph-mon[29707]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T15:28:29.534 INFO:teuthology.orchestra.run.smithi106.stdout:Created osd(s) 5 on host 'smithi106' 2024-04-02T15:28:30.094 DEBUG:teuthology.orchestra.run.smithi106:osd.5> sudo journalctl -f -n 0 -u ceph-0400842c-f105-11ee-b647-cb9ed24678a4@osd.5.service 2024-04-02T15:28:30.096 INFO:tasks.cephadm:Deploying osd.6 on smithi106 with /dev/vg_nvme/lv_2... 2024-04-02T15:28:30.096 DEBUG:teuthology.orchestra.run.smithi106:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-02T15:28:30.735 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:30 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:30.735 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:30 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:28:30.736 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:30 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:30.736 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:30 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:28:30.736 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:30 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:28:30.736 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:30 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:28:30.736 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:30 smithi106 ceph-mon[25843]: Adjusting osd_memory_target on smithi106 to 8032M 2024-04-02T15:28:30.736 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:30 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:30.736 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:30 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:30.736 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:30 smithi106 ceph-mon[25843]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T15:28:30.736 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:30 smithi106 ceph-mon[25843]: from='osd.5 [v2:172.21.15.106:6808/2879909052,v1:172.21.15.106:6809/2879909052]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-02T15:28:30.736 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:30 smithi106 ceph-mon[25843]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-02T15:28:30.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:30 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:30.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:30 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:28:30.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:30 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:30.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:30 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:28:30.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:30 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:28:30.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:30 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:28:30.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:30 smithi087 ceph-mon[24900]: Adjusting osd_memory_target on smithi106 to 8032M 2024-04-02T15:28:30.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:30 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:30.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:30 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:30.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:30 smithi087 ceph-mon[24900]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T15:28:30.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:30 smithi087 ceph-mon[24900]: from='osd.5 [v2:172.21.15.106:6808/2879909052,v1:172.21.15.106:6809/2879909052]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-02T15:28:30.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:30 smithi087 ceph-mon[24900]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-02T15:28:30.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:30 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:30.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:30 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:28:30.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:30 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:30.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:30 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:28:30.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:30 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:28:30.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:30 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:28:30.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:30 smithi087 ceph-mon[29707]: Adjusting osd_memory_target on smithi106 to 8032M 2024-04-02T15:28:30.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:30 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:30.872 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:30 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:30.872 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:30 smithi087 ceph-mon[29707]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T15:28:30.872 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:30 smithi087 ceph-mon[29707]: from='osd.5 [v2:172.21.15.106:6808/2879909052,v1:172.21.15.106:6809/2879909052]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-02T15:28:30.872 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:30 smithi087 ceph-mon[29707]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-02T15:28:31.791 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:31 smithi106 ceph-mon[25843]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-02T15:28:31.791 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:31 smithi106 ceph-mon[25843]: osdmap e37: 6 total, 5 up, 6 in 2024-04-02T15:28:31.792 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:31 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T15:28:31.792 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:31 smithi106 ceph-mon[25843]: from='osd.5 [v2:172.21.15.106:6808/2879909052,v1:172.21.15.106:6809/2879909052]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi106", "root=default"]}]: dispatch 2024-04-02T15:28:31.792 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:31 smithi106 ceph-mon[25843]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi106", "root=default"]}]: dispatch 2024-04-02T15:28:31.792 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:28:31 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5[33427]: 2024-04-02T15:28:31.571+0000 7fe3163e3700 -1 osd.5 0 waiting for initial osdmap 2024-04-02T15:28:31.792 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:28:31 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5[33427]: 2024-04-02T15:28:31.580+0000 7fe31257d700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T15:28:31.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:31 smithi087 ceph-mon[24900]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-02T15:28:31.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:31 smithi087 ceph-mon[24900]: osdmap e37: 6 total, 5 up, 6 in 2024-04-02T15:28:31.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:31 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T15:28:31.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:31 smithi087 ceph-mon[24900]: from='osd.5 [v2:172.21.15.106:6808/2879909052,v1:172.21.15.106:6809/2879909052]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi106", "root=default"]}]: dispatch 2024-04-02T15:28:31.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:31 smithi087 ceph-mon[24900]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi106", "root=default"]}]: dispatch 2024-04-02T15:28:31.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:31 smithi087 ceph-mon[29707]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-02T15:28:31.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:31 smithi087 ceph-mon[29707]: osdmap e37: 6 total, 5 up, 6 in 2024-04-02T15:28:31.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:31 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T15:28:31.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:31 smithi087 ceph-mon[29707]: from='osd.5 [v2:172.21.15.106:6808/2879909052,v1:172.21.15.106:6809/2879909052]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi106", "root=default"]}]: dispatch 2024-04-02T15:28:31.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:31 smithi087 ceph-mon[29707]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi106", "root=default"]}]: dispatch 2024-04-02T15:28:32.107 INFO:teuthology.orchestra.run.smithi106.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-02T15:28:32.107 INFO:teuthology.orchestra.run.smithi106.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-02T15:28:32.107 INFO:teuthology.orchestra.run.smithi106.stderr: stderr: 10+0 records in 2024-04-02T15:28:32.107 INFO:teuthology.orchestra.run.smithi106.stderr:10+0 records out 2024-04-02T15:28:32.107 INFO:teuthology.orchestra.run.smithi106.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0127977 s, 819 MB/s 2024-04-02T15:28:32.107 INFO:teuthology.orchestra.run.smithi106.stderr:--> Zapping successful for: 2024-04-02T15:28:32.668 DEBUG:teuthology.orchestra.run.smithi106:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi106:vg_nvme/lv_2 2024-04-02T15:28:32.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:32 smithi106 ceph-mon[25843]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi106", "root=default"]}]': finished 2024-04-02T15:28:32.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:32 smithi106 ceph-mon[25843]: osdmap e38: 6 total, 5 up, 6 in 2024-04-02T15:28:32.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:32 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T15:28:32.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:32 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T15:28:32.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:32 smithi106 ceph-mon[25843]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T15:28:32.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:32 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T15:28:32.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:32 smithi087 ceph-mon[24900]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi106", "root=default"]}]': finished 2024-04-02T15:28:32.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:32 smithi087 ceph-mon[24900]: osdmap e38: 6 total, 5 up, 6 in 2024-04-02T15:28:32.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:32 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T15:28:32.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:32 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T15:28:32.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:32 smithi087 ceph-mon[24900]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T15:28:32.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:32 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T15:28:32.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:32 smithi087 ceph-mon[29707]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi106", "root=default"]}]': finished 2024-04-02T15:28:32.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:32 smithi087 ceph-mon[29707]: osdmap e38: 6 total, 5 up, 6 in 2024-04-02T15:28:32.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:32 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T15:28:32.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:32 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T15:28:32.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:32 smithi087 ceph-mon[29707]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-02T15:28:32.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:32 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T15:28:33.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:33 smithi106 ceph-mon[25843]: purged_snaps scrub starts 2024-04-02T15:28:33.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:33 smithi106 ceph-mon[25843]: purged_snaps scrub ok 2024-04-02T15:28:33.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:33 smithi106 ceph-mon[25843]: osd.5 [v2:172.21.15.106:6808/2879909052,v1:172.21.15.106:6809/2879909052] boot 2024-04-02T15:28:33.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:33 smithi106 ceph-mon[25843]: osdmap e39: 6 total, 6 up, 6 in 2024-04-02T15:28:33.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:33 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T15:28:33.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:33 smithi087 ceph-mon[24900]: purged_snaps scrub starts 2024-04-02T15:28:33.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:33 smithi087 ceph-mon[24900]: purged_snaps scrub ok 2024-04-02T15:28:33.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:33 smithi087 ceph-mon[24900]: osd.5 [v2:172.21.15.106:6808/2879909052,v1:172.21.15.106:6809/2879909052] boot 2024-04-02T15:28:33.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:33 smithi087 ceph-mon[24900]: osdmap e39: 6 total, 6 up, 6 in 2024-04-02T15:28:33.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:33 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T15:28:33.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:33 smithi087 ceph-mon[29707]: purged_snaps scrub starts 2024-04-02T15:28:33.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:33 smithi087 ceph-mon[29707]: purged_snaps scrub ok 2024-04-02T15:28:33.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:33 smithi087 ceph-mon[29707]: osd.5 [v2:172.21.15.106:6808/2879909052,v1:172.21.15.106:6809/2879909052] boot 2024-04-02T15:28:33.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:33 smithi087 ceph-mon[29707]: osdmap e39: 6 total, 6 up, 6 in 2024-04-02T15:28:33.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:33 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T15:28:34.812 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:34 smithi106 ceph-mon[25843]: osdmap e40: 6 total, 6 up, 6 in 2024-04-02T15:28:34.812 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:34 smithi106 ceph-mon[25843]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:28:34.812 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:34 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T15:28:34.812 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:34 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T15:28:34.812 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:34 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:34.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:34 smithi087 ceph-mon[24900]: osdmap e40: 6 total, 6 up, 6 in 2024-04-02T15:28:34.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:34 smithi087 ceph-mon[24900]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:28:34.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:34 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T15:28:34.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:34 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T15:28:34.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:34 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:34.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:34 smithi087 ceph-mon[29707]: osdmap e40: 6 total, 6 up, 6 in 2024-04-02T15:28:34.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:34 smithi087 ceph-mon[29707]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:28:34.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:34 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T15:28:34.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:34 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T15:28:34.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:34 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:35.812 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:35 smithi106 ceph-mon[25843]: from='client.24269 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi106:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:28:35.812 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:35 smithi106 ceph-mon[25843]: osdmap e41: 6 total, 6 up, 6 in 2024-04-02T15:28:35.869 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:35 smithi087 ceph-mon[24900]: from='client.24269 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi106:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:28:35.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:35 smithi087 ceph-mon[24900]: osdmap e41: 6 total, 6 up, 6 in 2024-04-02T15:28:35.870 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:28:35 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:28:35] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:28:35.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:35 smithi087 ceph-mon[29707]: from='client.24269 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi106:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:28:35.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:35 smithi087 ceph-mon[29707]: osdmap e41: 6 total, 6 up, 6 in 2024-04-02T15:28:36.690 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:36 smithi106 ceph-mon[25843]: osdmap e42: 6 total, 6 up, 6 in 2024-04-02T15:28:36.690 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:36 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:36.690 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:36 smithi106 ceph-mon[25843]: Detected new or changed devices on smithi106 2024-04-02T15:28:36.690 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:36 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:36.690 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:36 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:28:36.690 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:36 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:36.690 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:36 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:28:36.690 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:36 smithi106 ceph-mon[25843]: pgmap v134: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:28:36.690 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:36 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:36.690 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:36 smithi106 ceph-mon[25843]: from='client.? 172.21.15.106:0/1515342802' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "391fb1af-ab61-4a21-8bf3-da6f0117756e"}]: dispatch 2024-04-02T15:28:36.691 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:36 smithi106 ceph-mon[25843]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "391fb1af-ab61-4a21-8bf3-da6f0117756e"}]: dispatch 2024-04-02T15:28:36.691 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:36 smithi106 ceph-mon[25843]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "391fb1af-ab61-4a21-8bf3-da6f0117756e"}]': finished 2024-04-02T15:28:36.691 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:36 smithi106 ceph-mon[25843]: osdmap e43: 7 total, 6 up, 7 in 2024-04-02T15:28:36.691 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:36 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T15:28:36.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:36 smithi087 ceph-mon[24900]: osdmap e42: 6 total, 6 up, 6 in 2024-04-02T15:28:36.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:36 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:36.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:36 smithi087 ceph-mon[24900]: Detected new or changed devices on smithi106 2024-04-02T15:28:36.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:36 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:36.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:36 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:28:36.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:36 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:36.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:36 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:28:36.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:36 smithi087 ceph-mon[24900]: pgmap v134: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:28:36.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:36 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:36.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:36 smithi087 ceph-mon[24900]: from='client.? 172.21.15.106:0/1515342802' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "391fb1af-ab61-4a21-8bf3-da6f0117756e"}]: dispatch 2024-04-02T15:28:36.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:36 smithi087 ceph-mon[24900]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "391fb1af-ab61-4a21-8bf3-da6f0117756e"}]: dispatch 2024-04-02T15:28:36.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:36 smithi087 ceph-mon[24900]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "391fb1af-ab61-4a21-8bf3-da6f0117756e"}]': finished 2024-04-02T15:28:36.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:36 smithi087 ceph-mon[24900]: osdmap e43: 7 total, 6 up, 7 in 2024-04-02T15:28:36.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:36 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T15:28:36.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:36 smithi087 ceph-mon[29707]: osdmap e42: 6 total, 6 up, 6 in 2024-04-02T15:28:36.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:36 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:36.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:36 smithi087 ceph-mon[29707]: Detected new or changed devices on smithi106 2024-04-02T15:28:36.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:36 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:36.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:36 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:28:36.872 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:36 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:36.872 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:36 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:28:36.872 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:36 smithi087 ceph-mon[29707]: pgmap v134: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:28:36.872 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:36 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:36.872 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:36 smithi087 ceph-mon[29707]: from='client.? 172.21.15.106:0/1515342802' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "391fb1af-ab61-4a21-8bf3-da6f0117756e"}]: dispatch 2024-04-02T15:28:36.872 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:36 smithi087 ceph-mon[29707]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "391fb1af-ab61-4a21-8bf3-da6f0117756e"}]: dispatch 2024-04-02T15:28:36.872 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:36 smithi087 ceph-mon[29707]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "391fb1af-ab61-4a21-8bf3-da6f0117756e"}]': finished 2024-04-02T15:28:36.872 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:36 smithi087 ceph-mon[29707]: osdmap e43: 7 total, 6 up, 7 in 2024-04-02T15:28:36.872 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:36 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T15:28:37.599 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:28:37 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:28:37] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:28:37.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:37 smithi106 ceph-mon[25843]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-04-02T15:28:37.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:37 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:37.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:37 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:37.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:37 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:28:37.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:37 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:37.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:37 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:28:37.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:37 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:37.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:37 smithi106 ceph-mon[25843]: from='client.? 172.21.15.106:0/2770946166' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T15:28:37.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:37 smithi087 ceph-mon[24900]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-04-02T15:28:37.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:37 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:37.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:37 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:37.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:37 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:28:37.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:37 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:37.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:37 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:28:37.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:37 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:37.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:37 smithi087 ceph-mon[24900]: from='client.? 172.21.15.106:0/2770946166' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T15:28:37.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:37 smithi087 ceph-mon[29707]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-04-02T15:28:37.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:37 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:37.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:37 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:37.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:37 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:28:37.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:37 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:37.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:37 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:28:37.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:37 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:37.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:37 smithi087 ceph-mon[29707]: from='client.? 172.21.15.106:0/2770946166' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T15:28:38.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:38 smithi106 ceph-mon[25843]: pgmap v136: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:28:38.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:38 smithi106 ceph-mon[25843]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-04-02T15:28:38.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:38 smithi106 ceph-mon[25843]: Cluster is now healthy 2024-04-02T15:28:38.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:38 smithi087 ceph-mon[24900]: pgmap v136: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:28:38.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:38 smithi087 ceph-mon[24900]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-04-02T15:28:38.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:38 smithi087 ceph-mon[24900]: Cluster is now healthy 2024-04-02T15:28:38.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:38 smithi087 ceph-mon[29707]: pgmap v136: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:28:38.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:38 smithi087 ceph-mon[29707]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-04-02T15:28:38.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:38 smithi087 ceph-mon[29707]: Cluster is now healthy 2024-04-02T15:28:40.299 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:40 smithi106 ceph-mon[25843]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:28:40.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:40 smithi087 ceph-mon[24900]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:28:40.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:40 smithi087 ceph-mon[29707]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:28:42.554 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:42 smithi106 ceph-mon[25843]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:28:42.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:42 smithi087 ceph-mon[29707]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:28:42.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:42 smithi087 ceph-mon[24900]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:28:44.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:44 smithi106 ceph-mon[25843]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:28:44.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:44 smithi087 ceph-mon[24900]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:28:44.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:44 smithi087 ceph-mon[29707]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:28:45.120 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:28:44 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:28:44] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:28:45.365 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:45 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-02T15:28:45.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:45 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:45.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:45 smithi106 ceph-mon[25843]: Deploying daemon osd.6 on smithi106 2024-04-02T15:28:45.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:45 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:45.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:45 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:45.594 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:45 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-02T15:28:45.594 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:45 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:45.595 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:45 smithi087 ceph-mon[24900]: Deploying daemon osd.6 on smithi106 2024-04-02T15:28:45.595 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:45 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:45.595 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:45 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:45.595 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:45 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-02T15:28:45.595 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:45 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:45.595 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:45 smithi087 ceph-mon[29707]: Deploying daemon osd.6 on smithi106 2024-04-02T15:28:45.595 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:45 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:45.595 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:45 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:46.995 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:46 smithi106 ceph-mon[25843]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:28:47.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:46 smithi087 ceph-mon[29707]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:28:47.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:46 smithi087 ceph-mon[24900]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:28:48.969 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:48 smithi106 ceph-mon[25843]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:28:49.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:48 smithi087 ceph-mon[24900]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:28:49.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:48 smithi087 ceph-mon[29707]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:28:50.526 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:50 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:50.526 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:50 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:28:50.526 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:50 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:50.526 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:50 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:28:50.526 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:50 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:50.526 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:50 smithi106 ceph-mon[25843]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:28:50.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:50 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:50.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:50 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:28:50.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:50 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:50.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:50 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:28:50.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:50 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:50.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:50 smithi087 ceph-mon[24900]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:28:50.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:50 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:50.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:50 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:28:50.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:50 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:50.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:50 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:28:50.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:50 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:50.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:50 smithi087 ceph-mon[29707]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:28:51.915 INFO:teuthology.orchestra.run.smithi106.stdout:Created osd(s) 6 on host 'smithi106' 2024-04-02T15:28:52.509 DEBUG:teuthology.orchestra.run.smithi106:osd.6> sudo journalctl -f -n 0 -u ceph-0400842c-f105-11ee-b647-cb9ed24678a4@osd.6.service 2024-04-02T15:28:52.512 INFO:tasks.cephadm:Deploying osd.7 on smithi106 with /dev/vg_nvme/lv_1... 2024-04-02T15:28:52.513 DEBUG:teuthology.orchestra.run.smithi106:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-02T15:28:53.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:52 smithi106 ceph-mon[25843]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:28:53.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:52 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:53.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:52 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:28:53.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:52 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:53.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:52 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:28:53.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:52 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:28:53.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:52 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:28:53.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:52 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:28:53.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:52 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:53.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:52 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:53.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:52 smithi106 ceph-mon[25843]: from='osd.6 [v2:172.21.15.106:6816/125323095,v1:172.21.15.106:6817/125323095]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-02T15:28:53.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:52 smithi106 ceph-mon[25843]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-02T15:28:53.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:52 smithi087 ceph-mon[24900]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:28:53.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:52 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:53.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:52 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:28:53.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:52 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:53.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:52 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:28:53.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:52 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:28:53.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:52 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:28:53.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:52 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:28:53.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:52 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:53.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:52 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:53.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:52 smithi087 ceph-mon[24900]: from='osd.6 [v2:172.21.15.106:6816/125323095,v1:172.21.15.106:6817/125323095]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-02T15:28:53.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:52 smithi087 ceph-mon[24900]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-02T15:28:53.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:52 smithi087 ceph-mon[29707]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:28:53.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:52 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:53.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:52 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:28:53.372 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:52 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:53.372 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:52 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:28:53.372 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:52 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:28:53.372 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:52 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:28:53.372 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:52 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:28:53.372 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:52 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:53.372 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:52 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:53.372 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:52 smithi087 ceph-mon[29707]: from='osd.6 [v2:172.21.15.106:6816/125323095,v1:172.21.15.106:6817/125323095]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-02T15:28:53.372 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:52 smithi087 ceph-mon[29707]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-02T15:28:54.070 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:53 smithi106 ceph-mon[25843]: Adjusting osd_memory_target on smithi106 to 5354M 2024-04-02T15:28:54.070 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:53 smithi106 ceph-mon[25843]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-02T15:28:54.070 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:53 smithi106 ceph-mon[25843]: osdmap e44: 7 total, 6 up, 7 in 2024-04-02T15:28:54.071 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:53 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T15:28:54.071 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:53 smithi106 ceph-mon[25843]: from='osd.6 [v2:172.21.15.106:6816/125323095,v1:172.21.15.106:6817/125323095]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi106", "root=default"]}]: dispatch 2024-04-02T15:28:54.071 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:53 smithi106 ceph-mon[25843]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi106", "root=default"]}]: dispatch 2024-04-02T15:28:54.071 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:28:53 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:28:53.979+0000 7f2d20514700 -1 osd.6 0 waiting for initial osdmap 2024-04-02T15:28:54.071 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:28:53 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:28:53.986+0000 7f2d1aeab700 -1 osd.6 45 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T15:28:54.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:53 smithi087 ceph-mon[24900]: Adjusting osd_memory_target on smithi106 to 5354M 2024-04-02T15:28:54.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:53 smithi087 ceph-mon[24900]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-02T15:28:54.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:53 smithi087 ceph-mon[24900]: osdmap e44: 7 total, 6 up, 7 in 2024-04-02T15:28:54.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:53 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T15:28:54.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:53 smithi087 ceph-mon[24900]: from='osd.6 [v2:172.21.15.106:6816/125323095,v1:172.21.15.106:6817/125323095]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi106", "root=default"]}]: dispatch 2024-04-02T15:28:54.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:53 smithi087 ceph-mon[24900]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi106", "root=default"]}]: dispatch 2024-04-02T15:28:54.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:53 smithi087 ceph-mon[29707]: Adjusting osd_memory_target on smithi106 to 5354M 2024-04-02T15:28:54.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:53 smithi087 ceph-mon[29707]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-02T15:28:54.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:53 smithi087 ceph-mon[29707]: osdmap e44: 7 total, 6 up, 7 in 2024-04-02T15:28:54.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:53 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T15:28:54.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:53 smithi087 ceph-mon[29707]: from='osd.6 [v2:172.21.15.106:6816/125323095,v1:172.21.15.106:6817/125323095]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi106", "root=default"]}]: dispatch 2024-04-02T15:28:54.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:53 smithi087 ceph-mon[29707]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi106", "root=default"]}]: dispatch 2024-04-02T15:28:54.445 INFO:teuthology.orchestra.run.smithi106.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-02T15:28:54.445 INFO:teuthology.orchestra.run.smithi106.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-02T15:28:54.446 INFO:teuthology.orchestra.run.smithi106.stderr: stderr: 10+0 records in 2024-04-02T15:28:54.446 INFO:teuthology.orchestra.run.smithi106.stderr:10+0 records out 2024-04-02T15:28:54.446 INFO:teuthology.orchestra.run.smithi106.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0124508 s, 842 MB/s 2024-04-02T15:28:54.446 INFO:teuthology.orchestra.run.smithi106.stderr:--> Zapping successful for: 2024-04-02T15:28:55.059 DEBUG:teuthology.orchestra.run.smithi106:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi106:vg_nvme/lv_1 2024-04-02T15:28:55.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:54 smithi106 ceph-mon[25843]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:28:55.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:54 smithi106 ceph-mon[25843]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi106", "root=default"]}]': finished 2024-04-02T15:28:55.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:54 smithi106 ceph-mon[25843]: osdmap e45: 7 total, 6 up, 7 in 2024-04-02T15:28:55.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:54 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T15:28:55.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:54 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T15:28:55.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:54 smithi106 ceph-mon[25843]: osd.6 [v2:172.21.15.106:6816/125323095,v1:172.21.15.106:6817/125323095] boot 2024-04-02T15:28:55.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:54 smithi106 ceph-mon[25843]: osdmap e46: 7 total, 7 up, 7 in 2024-04-02T15:28:55.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:54 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T15:28:55.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:54 smithi087 ceph-mon[24900]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:28:55.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:54 smithi087 ceph-mon[24900]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi106", "root=default"]}]': finished 2024-04-02T15:28:55.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:54 smithi087 ceph-mon[24900]: osdmap e45: 7 total, 6 up, 7 in 2024-04-02T15:28:55.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:54 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T15:28:55.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:54 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T15:28:55.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:54 smithi087 ceph-mon[24900]: osd.6 [v2:172.21.15.106:6816/125323095,v1:172.21.15.106:6817/125323095] boot 2024-04-02T15:28:55.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:54 smithi087 ceph-mon[24900]: osdmap e46: 7 total, 7 up, 7 in 2024-04-02T15:28:55.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:54 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T15:28:55.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:54 smithi087 ceph-mon[29707]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-02T15:28:55.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:54 smithi087 ceph-mon[29707]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi106", "root=default"]}]': finished 2024-04-02T15:28:55.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:54 smithi087 ceph-mon[29707]: osdmap e45: 7 total, 6 up, 7 in 2024-04-02T15:28:55.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:54 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T15:28:55.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:54 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T15:28:55.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:54 smithi087 ceph-mon[29707]: osd.6 [v2:172.21.15.106:6816/125323095,v1:172.21.15.106:6817/125323095] boot 2024-04-02T15:28:55.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:54 smithi087 ceph-mon[29707]: osdmap e46: 7 total, 7 up, 7 in 2024-04-02T15:28:55.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:54 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T15:28:56.235 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:56 smithi106 ceph-mon[25843]: purged_snaps scrub starts 2024-04-02T15:28:56.236 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:56 smithi106 ceph-mon[25843]: purged_snaps scrub ok 2024-04-02T15:28:56.236 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:56 smithi106 ceph-mon[25843]: osdmap e47: 7 total, 7 up, 7 in 2024-04-02T15:28:56.236 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:56 smithi106 ceph-mon[25843]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-02T15:28:56.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:56 smithi087 ceph-mon[24900]: purged_snaps scrub starts 2024-04-02T15:28:56.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:56 smithi087 ceph-mon[24900]: purged_snaps scrub ok 2024-04-02T15:28:56.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:56 smithi087 ceph-mon[24900]: osdmap e47: 7 total, 7 up, 7 in 2024-04-02T15:28:56.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:56 smithi087 ceph-mon[24900]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-02T15:28:56.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:56 smithi087 ceph-mon[29707]: purged_snaps scrub starts 2024-04-02T15:28:56.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:56 smithi087 ceph-mon[29707]: purged_snaps scrub ok 2024-04-02T15:28:56.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:56 smithi087 ceph-mon[29707]: osdmap e47: 7 total, 7 up, 7 in 2024-04-02T15:28:56.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:56 smithi087 ceph-mon[29707]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-02T15:28:57.303 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:57 smithi106 ceph-mon[25843]: osdmap e48: 7 total, 7 up, 7 in 2024-04-02T15:28:57.303 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:57 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T15:28:57.303 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:57 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T15:28:57.303 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:57 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:57.620 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:28:57 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:28:57] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:28:57.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:57 smithi087 ceph-mon[24900]: osdmap e48: 7 total, 7 up, 7 in 2024-04-02T15:28:57.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:57 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T15:28:57.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:57 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T15:28:57.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:57 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:57.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:57 smithi087 ceph-mon[29707]: osdmap e48: 7 total, 7 up, 7 in 2024-04-02T15:28:57.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:57 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T15:28:57.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:57 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T15:28:57.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:57 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:58.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:58 smithi106 ceph-mon[25843]: from='client.24296 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi106:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:28:58.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:58 smithi106 ceph-mon[25843]: osdmap e49: 7 total, 7 up, 7 in 2024-04-02T15:28:58.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:58 smithi106 ceph-mon[25843]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-02T15:28:58.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:58 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:58.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:58 smithi106 ceph-mon[25843]: Detected new or changed devices on smithi106 2024-04-02T15:28:58.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:58 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:58.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:58 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:28:58.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:58 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:58.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:58 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:28:58.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:58 smithi106 ceph-mon[25843]: pgmap v153: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-02T15:28:58.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:58 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:58.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:58 smithi087 ceph-mon[29707]: from='client.24296 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi106:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:28:58.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:58 smithi087 ceph-mon[29707]: osdmap e49: 7 total, 7 up, 7 in 2024-04-02T15:28:58.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:58 smithi087 ceph-mon[29707]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-02T15:28:58.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:58 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:58.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:58 smithi087 ceph-mon[29707]: Detected new or changed devices on smithi106 2024-04-02T15:28:58.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:58 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:58.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:58 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:28:58.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:58 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:58.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:58 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:28:58.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:58 smithi087 ceph-mon[29707]: pgmap v153: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-02T15:28:58.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:58 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:58.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:58 smithi087 ceph-mon[24900]: from='client.24296 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi106:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:28:58.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:58 smithi087 ceph-mon[24900]: osdmap e49: 7 total, 7 up, 7 in 2024-04-02T15:28:58.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:58 smithi087 ceph-mon[24900]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-02T15:28:58.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:58 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:58.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:58 smithi087 ceph-mon[24900]: Detected new or changed devices on smithi106 2024-04-02T15:28:58.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:58 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:58.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:58 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:28:58.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:58 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:28:58.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:58 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:28:58.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:58 smithi087 ceph-mon[24900]: pgmap v153: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-02T15:28:58.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:58 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:28:59.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:59 smithi106 ceph-mon[25843]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-04-02T15:28:59.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:28:59 smithi106 ceph-mon[25843]: osdmap e50: 7 total, 7 up, 7 in 2024-04-02T15:28:59.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:59 smithi087 ceph-mon[24900]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-04-02T15:28:59.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:28:59 smithi087 ceph-mon[24900]: osdmap e50: 7 total, 7 up, 7 in 2024-04-02T15:28:59.620 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:28:59 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:28:59] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:28:59.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:59 smithi087 ceph-mon[29707]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-04-02T15:28:59.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:28:59 smithi087 ceph-mon[29707]: osdmap e50: 7 total, 7 up, 7 in 2024-04-02T15:29:00.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:00 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:00.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:00 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:00.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:00 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:29:00.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:00 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:29:00.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:00 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:29:00.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:00 smithi106 ceph-mon[25843]: pgmap v155: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-02T15:29:00.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:00 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:00.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:00 smithi106 ceph-mon[25843]: from='client.? 172.21.15.106:0/3274037531' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "780bffc8-5404-40eb-860f-26c6e0c773b6"}]: dispatch 2024-04-02T15:29:00.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:00 smithi106 ceph-mon[25843]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "780bffc8-5404-40eb-860f-26c6e0c773b6"}]: dispatch 2024-04-02T15:29:00.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:00 smithi106 ceph-mon[25843]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "780bffc8-5404-40eb-860f-26c6e0c773b6"}]': finished 2024-04-02T15:29:00.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:00 smithi106 ceph-mon[25843]: osdmap e51: 8 total, 7 up, 8 in 2024-04-02T15:29:00.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:00 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:29:00.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:00 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:00.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:00 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:00.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:00 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:29:00.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:00 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:29:00.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:00 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:29:00.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:00 smithi087 ceph-mon[24900]: pgmap v155: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-02T15:29:00.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:00 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:00.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:00 smithi087 ceph-mon[24900]: from='client.? 172.21.15.106:0/3274037531' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "780bffc8-5404-40eb-860f-26c6e0c773b6"}]: dispatch 2024-04-02T15:29:00.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:00 smithi087 ceph-mon[24900]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "780bffc8-5404-40eb-860f-26c6e0c773b6"}]: dispatch 2024-04-02T15:29:00.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:00 smithi087 ceph-mon[24900]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "780bffc8-5404-40eb-860f-26c6e0c773b6"}]': finished 2024-04-02T15:29:00.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:00 smithi087 ceph-mon[24900]: osdmap e51: 8 total, 7 up, 8 in 2024-04-02T15:29:00.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:00 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:29:00.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:00 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:00.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:00 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:00.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:00 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:29:00.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:00 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:29:00.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:00 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:29:00.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:00 smithi087 ceph-mon[29707]: pgmap v155: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-02T15:29:00.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:00 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:00.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:00 smithi087 ceph-mon[29707]: from='client.? 172.21.15.106:0/3274037531' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "780bffc8-5404-40eb-860f-26c6e0c773b6"}]: dispatch 2024-04-02T15:29:00.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:00 smithi087 ceph-mon[29707]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "780bffc8-5404-40eb-860f-26c6e0c773b6"}]: dispatch 2024-04-02T15:29:00.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:00 smithi087 ceph-mon[29707]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "780bffc8-5404-40eb-860f-26c6e0c773b6"}]': finished 2024-04-02T15:29:00.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:00 smithi087 ceph-mon[29707]: osdmap e51: 8 total, 7 up, 8 in 2024-04-02T15:29:00.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:00 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:29:01.615 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:01 smithi106 ceph-mon[25843]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-04-02T15:29:01.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:01 smithi106 ceph-mon[25843]: Cluster is now healthy 2024-04-02T15:29:01.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:01 smithi106 ceph-mon[25843]: from='client.? 172.21.15.106:0/2878609663' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T15:29:01.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:01 smithi087 ceph-mon[24900]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-04-02T15:29:01.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:01 smithi087 ceph-mon[24900]: Cluster is now healthy 2024-04-02T15:29:01.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:01 smithi087 ceph-mon[24900]: from='client.? 172.21.15.106:0/2878609663' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T15:29:01.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:01 smithi087 ceph-mon[29707]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-04-02T15:29:01.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:01 smithi087 ceph-mon[29707]: Cluster is now healthy 2024-04-02T15:29:01.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:01 smithi087 ceph-mon[29707]: from='client.? 172.21.15.106:0/2878609663' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T15:29:02.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:02 smithi106 ceph-mon[25843]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 127 KiB/s, 0 objects/s recovering 2024-04-02T15:29:02.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:02 smithi087 ceph-mon[24900]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 127 KiB/s, 0 objects/s recovering 2024-04-02T15:29:02.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:02 smithi087 ceph-mon[29707]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 127 KiB/s, 0 objects/s recovering 2024-04-02T15:29:04.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:04 smithi106 ceph-mon[25843]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-02T15:29:04.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:04 smithi087 ceph-mon[24900]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-02T15:29:04.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:04 smithi087 ceph-mon[29707]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-02T15:29:05.120 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:04 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:29:04] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:29:06.080 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:05 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:06.081 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:05 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:06.081 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:05 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:06.081 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:05 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:06.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:05 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:06.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:05 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:07.072 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:06 smithi106 ceph-mon[25843]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 66 KiB/s, 0 objects/s recovering 2024-04-02T15:29:07.072 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:06 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-02T15:29:07.072 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:06 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:29:07.113 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:06 smithi087 ceph-mon[24900]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 66 KiB/s, 0 objects/s recovering 2024-04-02T15:29:07.113 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:06 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-02T15:29:07.113 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:06 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:29:07.114 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:06 smithi087 ceph-mon[29707]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 66 KiB/s, 0 objects/s recovering 2024-04-02T15:29:07.114 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:06 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-02T15:29:07.114 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:06 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:29:08.115 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:07 smithi106 ceph-mon[25843]: Deploying daemon osd.7 on smithi106 2024-04-02T15:29:08.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:07 smithi087 ceph-mon[24900]: Deploying daemon osd.7 on smithi106 2024-04-02T15:29:08.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:07 smithi087 ceph-mon[29707]: Deploying daemon osd.7 on smithi106 2024-04-02T15:29:09.107 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:08 smithi106 ceph-mon[25843]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 57 KiB/s, 0 objects/s recovering 2024-04-02T15:29:09.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:08 smithi087 ceph-mon[24900]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 57 KiB/s, 0 objects/s recovering 2024-04-02T15:29:09.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:08 smithi087 ceph-mon[29707]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 57 KiB/s, 0 objects/s recovering 2024-04-02T15:29:10.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:10 smithi106 ceph-mon[25843]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-02T15:29:10.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:10 smithi087 ceph-mon[29707]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-02T15:29:10.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:10 smithi087 ceph-mon[24900]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-02T15:29:12.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:12 smithi106 ceph-mon[25843]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 44 KiB/s, 0 objects/s recovering 2024-04-02T15:29:12.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:12 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:12.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:12 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:29:12.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:12 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:29:12.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:12 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:29:12.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:12 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:12.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:12 smithi087 ceph-mon[24900]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 44 KiB/s, 0 objects/s recovering 2024-04-02T15:29:12.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:12 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:12.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:12 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:29:12.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:12 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:29:12.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:12 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:29:12.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:12 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:12.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:12 smithi087 ceph-mon[29707]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 44 KiB/s, 0 objects/s recovering 2024-04-02T15:29:12.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:12 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:12.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:12 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:29:12.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:12 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:29:12.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:12 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:29:12.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:12 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:14.182 INFO:teuthology.orchestra.run.smithi106.stdout:Created osd(s) 7 on host 'smithi106' 2024-04-02T15:29:14.842 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:14 smithi106 ceph-mon[25843]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T15:29:14.843 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:14 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:14.843 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:14 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:29:14.843 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:14 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:29:14.843 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:14 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:29:14.843 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:14 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:29:14.843 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:14 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:29:14.843 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:14 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:29:14.843 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:14 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:29:14.843 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:14 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:14.843 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:14 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:14.865 DEBUG:teuthology.orchestra.run.smithi106:osd.7> sudo journalctl -f -n 0 -u ceph-0400842c-f105-11ee-b647-cb9ed24678a4@osd.7.service 2024-04-02T15:29:14.868 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-04-02T15:29:14.868 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph osd stat -f json 2024-04-02T15:29:14.872 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:14 smithi087 ceph-mon[24900]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T15:29:14.872 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:14 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:14.872 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:14 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:29:14.872 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:14 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:29:14.872 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:14 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:29:14.872 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:14 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:29:14.872 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:14 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:29:14.872 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:14 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:29:14.872 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:14 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:29:14.872 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:14 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:14.872 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:14 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:14.873 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:14 smithi087 ceph-mon[29707]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T15:29:14.873 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:14 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:14.873 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:14 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:29:14.873 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:14 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:29:14.873 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:14 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:29:14.873 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:14 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:29:14.873 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:14 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:29:14.873 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:14 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:29:14.873 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:14 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:29:14.873 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:14 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:14.873 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:14 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:15.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:15 smithi106 ceph-mon[25843]: Adjusting osd_memory_target on smithi106 to 4016M 2024-04-02T15:29:15.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:15 smithi106 ceph-mon[25843]: from='osd.7 [v2:172.21.15.106:6824/256892273,v1:172.21.15.106:6825/256892273]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-02T15:29:15.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:15 smithi106 ceph-mon[25843]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-02T15:29:15.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:15 smithi087 ceph-mon[24900]: Adjusting osd_memory_target on smithi106 to 4016M 2024-04-02T15:29:15.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:15 smithi087 ceph-mon[24900]: from='osd.7 [v2:172.21.15.106:6824/256892273,v1:172.21.15.106:6825/256892273]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-02T15:29:15.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:15 smithi087 ceph-mon[24900]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-02T15:29:15.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:15 smithi087 ceph-mon[29707]: Adjusting osd_memory_target on smithi106 to 4016M 2024-04-02T15:29:15.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:15 smithi087 ceph-mon[29707]: from='osd.7 [v2:172.21.15.106:6824/256892273,v1:172.21.15.106:6825/256892273]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-02T15:29:15.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:15 smithi087 ceph-mon[29707]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-02T15:29:16.793 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-02T15:29:16.794 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:16 smithi087 ceph-mon[24900]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T15:29:16.794 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:16 smithi087 ceph-mon[24900]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-02T15:29:16.794 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:16 smithi087 ceph-mon[24900]: osdmap e52: 8 total, 7 up, 8 in 2024-04-02T15:29:16.794 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:16 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:29:16.794 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:16 smithi087 ceph-mon[24900]: from='osd.7 [v2:172.21.15.106:6824/256892273,v1:172.21.15.106:6825/256892273]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi106", "root=default"]}]: dispatch 2024-04-02T15:29:16.794 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:16 smithi087 ceph-mon[24900]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi106", "root=default"]}]: dispatch 2024-04-02T15:29:16.795 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:16 smithi087 ceph-mon[29707]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T15:29:16.795 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:16 smithi087 ceph-mon[29707]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-02T15:29:16.795 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:16 smithi087 ceph-mon[29707]: osdmap e52: 8 total, 7 up, 8 in 2024-04-02T15:29:16.795 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:16 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:29:16.795 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:16 smithi087 ceph-mon[29707]: from='osd.7 [v2:172.21.15.106:6824/256892273,v1:172.21.15.106:6825/256892273]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi106", "root=default"]}]: dispatch 2024-04-02T15:29:16.795 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:16 smithi087 ceph-mon[29707]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi106", "root=default"]}]: dispatch 2024-04-02T15:29:16.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:16 smithi106 ceph-mon[25843]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T15:29:16.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:16 smithi106 ceph-mon[25843]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-02T15:29:16.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:16 smithi106 ceph-mon[25843]: osdmap e52: 8 total, 7 up, 8 in 2024-04-02T15:29:16.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:16 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:29:16.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:16 smithi106 ceph-mon[25843]: from='osd.7 [v2:172.21.15.106:6824/256892273,v1:172.21.15.106:6825/256892273]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi106", "root=default"]}]: dispatch 2024-04-02T15:29:16.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:16 smithi106 ceph-mon[25843]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi106", "root=default"]}]: dispatch 2024-04-02T15:29:16.867 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:29:16 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:29:16.598+0000 7ff8cbb4b700 -1 osd.7 0 waiting for initial osdmap 2024-04-02T15:29:16.867 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:29:16 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:29:16.607+0000 7ff8c74e4700 -1 osd.7 53 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T15:29:17.529 INFO:teuthology.orchestra.run.smithi087.stdout:{"epoch":53,"num_osds":8,"num_up_osds":7,"osd_up_since":1712071734,"num_in_osds":8,"osd_in_since":1712071739,"num_remapped_pgs":0} 2024-04-02T15:29:17.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:17 smithi106 ceph-mon[25843]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi106", "root=default"]}]': finished 2024-04-02T15:29:17.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:17 smithi106 ceph-mon[25843]: osdmap e53: 8 total, 7 up, 8 in 2024-04-02T15:29:17.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:17 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:29:17.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:17 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:29:17.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:17 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/327577468' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-02T15:29:17.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:17 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:29:17.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:17 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:29:17.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:17 smithi087 ceph-mon[24900]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi106", "root=default"]}]': finished 2024-04-02T15:29:17.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:17 smithi087 ceph-mon[24900]: osdmap e53: 8 total, 7 up, 8 in 2024-04-02T15:29:17.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:17 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:29:17.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:17 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:29:17.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:17 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/327577468' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-02T15:29:17.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:17 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:29:17.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:17 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:29:17.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:17 smithi087 ceph-mon[29707]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi106", "root=default"]}]': finished 2024-04-02T15:29:17.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:17 smithi087 ceph-mon[29707]: osdmap e53: 8 total, 7 up, 8 in 2024-04-02T15:29:17.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:17 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:29:17.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:17 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:29:17.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:17 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/327577468' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-02T15:29:17.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:17 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:29:17.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:17 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:29:18.530 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph osd stat -f json 2024-04-02T15:29:18.848 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:18 smithi087 ceph-mon[29707]: purged_snaps scrub starts 2024-04-02T15:29:18.848 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:18 smithi087 ceph-mon[29707]: purged_snaps scrub ok 2024-04-02T15:29:18.848 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:18 smithi087 ceph-mon[29707]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T15:29:18.848 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:18 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:29:18.848 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:18 smithi087 ceph-mon[29707]: osd.7 [v2:172.21.15.106:6824/256892273,v1:172.21.15.106:6825/256892273] boot 2024-04-02T15:29:18.848 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:18 smithi087 ceph-mon[29707]: osdmap e54: 8 total, 8 up, 8 in 2024-04-02T15:29:18.848 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:18 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:29:18.848 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:18 smithi087 ceph-mon[29707]: osdmap e55: 8 total, 8 up, 8 in 2024-04-02T15:29:18.848 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:18 smithi087 ceph-mon[24900]: purged_snaps scrub starts 2024-04-02T15:29:18.849 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:18 smithi087 ceph-mon[24900]: purged_snaps scrub ok 2024-04-02T15:29:18.849 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:18 smithi087 ceph-mon[24900]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T15:29:18.849 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:18 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:29:18.849 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:18 smithi087 ceph-mon[24900]: osd.7 [v2:172.21.15.106:6824/256892273,v1:172.21.15.106:6825/256892273] boot 2024-04-02T15:29:18.849 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:18 smithi087 ceph-mon[24900]: osdmap e54: 8 total, 8 up, 8 in 2024-04-02T15:29:18.849 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:18 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:29:18.849 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:18 smithi087 ceph-mon[24900]: osdmap e55: 8 total, 8 up, 8 in 2024-04-02T15:29:18.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:18 smithi106 ceph-mon[25843]: purged_snaps scrub starts 2024-04-02T15:29:18.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:18 smithi106 ceph-mon[25843]: purged_snaps scrub ok 2024-04-02T15:29:18.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:18 smithi106 ceph-mon[25843]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-02T15:29:18.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:18 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:29:18.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:18 smithi106 ceph-mon[25843]: osd.7 [v2:172.21.15.106:6824/256892273,v1:172.21.15.106:6825/256892273] boot 2024-04-02T15:29:18.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:18 smithi106 ceph-mon[25843]: osdmap e54: 8 total, 8 up, 8 in 2024-04-02T15:29:18.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:18 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:29:18.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:18 smithi106 ceph-mon[25843]: osdmap e55: 8 total, 8 up, 8 in 2024-04-02T15:29:19.574 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:19 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:29:19] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:29:19.851 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:19 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:29:19] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:29:20.531 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-02T15:29:20.532 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:20 smithi087 ceph-mon[24900]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:20.532 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:20 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:20.532 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:20 smithi087 ceph-mon[24900]: Detected new or changed devices on smithi106 2024-04-02T15:29:20.532 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:20 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:20.532 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:20 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:29:20.532 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:20 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:29:20.532 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:20 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:29:20.532 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:20 smithi087 ceph-mon[24900]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:20.532 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:20 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:20.532 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:20 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:20.532 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:20 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:20.532 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:20 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:29:20.533 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:20 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:29:20.533 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:20 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:29:20.533 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:20 smithi087 ceph-mon[24900]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:20.533 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:20 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:20.533 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:20 smithi087 ceph-mon[24900]: osdmap e56: 8 total, 8 up, 8 in 2024-04-02T15:29:20.533 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:20 smithi087 ceph-mon[29707]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:20.533 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:20 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:20.533 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:20 smithi087 ceph-mon[29707]: Detected new or changed devices on smithi106 2024-04-02T15:29:20.533 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:20 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:20.533 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:20 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:29:20.533 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:20 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:29:20.533 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:20 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:29:20.533 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:20 smithi087 ceph-mon[29707]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:20.533 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:20 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:20.534 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:20 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:20.534 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:20 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:20.534 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:20 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:29:20.534 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:20 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:29:20.534 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:20 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:29:20.534 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:20 smithi087 ceph-mon[29707]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:20.534 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:20 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:20.534 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:20 smithi087 ceph-mon[29707]: osdmap e56: 8 total, 8 up, 8 in 2024-04-02T15:29:20.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:20 smithi106 ceph-mon[25843]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:20.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:20 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:20.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:20 smithi106 ceph-mon[25843]: Detected new or changed devices on smithi106 2024-04-02T15:29:20.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:20 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:20.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:20 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:29:20.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:20 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:29:20.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:20 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:29:20.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:20 smithi106 ceph-mon[25843]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:20.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:20 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:20.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:20 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:20.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:20 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:20.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:20 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:29:20.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:20 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:29:20.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:20 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:29:20.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:20 smithi106 ceph-mon[25843]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:20.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:20 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:20.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:20 smithi106 ceph-mon[25843]: osdmap e56: 8 total, 8 up, 8 in 2024-04-02T15:29:21.232 INFO:teuthology.orchestra.run.smithi087.stdout:{"epoch":56,"num_osds":8,"num_up_osds":8,"osd_up_since":1712071757,"num_in_osds":8,"osd_in_since":1712071739,"num_remapped_pgs":1} 2024-04-02T15:29:21.233 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-02T15:29:21.449 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:29:21.559 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:21 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/2602752749' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-02T15:29:21.559 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:21 smithi106 ceph-mon[25843]: osdmap e57: 8 total, 8 up, 8 in 2024-04-02T15:29:21.570 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:21 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/2602752749' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-02T15:29:21.570 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:21 smithi087 ceph-mon[29707]: osdmap e57: 8 total, 8 up, 8 in 2024-04-02T15:29:21.570 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:21 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/2602752749' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-02T15:29:21.570 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:21 smithi087 ceph-mon[24900]: osdmap e57: 8 total, 8 up, 8 in 2024-04-02T15:29:22.782 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:22 smithi087 ceph-mon[29707]: pgmap v175: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 173 KiB/s, 0 objects/s recovering 2024-04-02T15:29:22.783 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:22 smithi087 ceph-mon[29707]: osdmap e58: 8 total, 8 up, 8 in 2024-04-02T15:29:22.783 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:22 smithi087 ceph-mon[24900]: pgmap v175: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 173 KiB/s, 0 objects/s recovering 2024-04-02T15:29:22.783 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:22 smithi087 ceph-mon[24900]: osdmap e58: 8 total, 8 up, 8 in 2024-04-02T15:29:22.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:22 smithi106 ceph-mon[25843]: pgmap v175: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 173 KiB/s, 0 objects/s recovering 2024-04-02T15:29:22.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:22 smithi106 ceph-mon[25843]: osdmap e58: 8 total, 8 up, 8 in 2024-04-02T15:29:23.221 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-02T15:29:23.222 INFO:teuthology.orchestra.run.smithi087.stdout:{"epoch":58,"fsid":"0400842c-f105-11ee-b647-cb9ed24678a4","created":"2024-04-02T15:24:24.701153+0000","modified":"2024-04-02T15:29:21.602305+0000","last_up_change":"2024-04-02T15:29:17.597118+0000","last_in_change":"2024-04-02T15:28:59.589376+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-02T15:27:25.144519+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"13d89f7c-4e33-4521-a94a-ff0658e7dccf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6802","nonce":2380439245},{"type":"v1","addr":"172.21.15.87:6803","nonce":2380439245}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6804","nonce":2380439245},{"type":"v1","addr":"172.21.15.87:6805","nonce":2380439245}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6808","nonce":2380439245},{"type":"v1","addr":"172.21.15.87:6809","nonce":2380439245}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6806","nonce":2380439245},{"type":"v1","addr":"172.21.15.87:6807","nonce":2380439245}]},"public_addr":"172.21.15.87:6803/2380439245","cluster_addr":"172.21.15.87:6805/2380439245","heartbeat_back_addr":"172.21.15.87:6809/2380439245","heartbeat_front_addr":"172.21.15.87:6807/2380439245","state":["exists","up"]},{"osd":1,"uuid":"f8e09079-dd20-411b-a160-5d6906d1d7ed","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6810","nonce":1909345406},{"type":"v1","addr":"172.21.15.87:6811","nonce":1909345406}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6812","nonce":1909345406},{"type":"v1","addr":"172.21.15.87:6813","nonce":1909345406}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6816","nonce":1909345406},{"type":"v1","addr":"172.21.15.87:6817","nonce":1909345406}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6814","nonce":1909345406},{"type":"v1","addr":"172.21.15.87:6815","nonce":1909345406}]},"public_addr":"172.21.15.87:6811/1909345406","cluster_addr":"172.21.15.87:6813/1909345406","heartbeat_back_addr":"172.21.15.87:6817/1909345406","heartbeat_front_addr":"172.21.15.87:6815/1909345406","state":["exists","up"]},{"osd":2,"uuid":"55868ded-a2a8-48c7-8fb2-d452831755d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6818","nonce":2886422817},{"type":"v1","addr":"172.21.15.87:6819","nonce":2886422817}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6820","nonce":2886422817},{"type":"v1","addr":"172.21.15.87:6821","nonce":2886422817}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6824","nonce":2886422817},{"type":"v1","addr":"172.21.15.87:6825","nonce":2886422817}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6822","nonce":2886422817},{"type":"v1","addr":"172.21.15.87:6823","nonce":2886422817}]},"public_addr":"172.21.15.87:6819/2886422817","cluster_addr":"172.21.15.87:6821/2886422817","heartbeat_back_addr":"172.21.15.87:6825/2886422817","heartbeat_front_addr":"172.21.15.87:6823/2886422817","state":["exists","up"]},{"osd":3,"uuid":"fba679e6-6a92-4943-bbc9-3c446311dfc5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6826","nonce":1277310151},{"type":"v1","addr":"172.21.15.87:6827","nonce":1277310151}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6828","nonce":1277310151},{"type":"v1","addr":"172.21.15.87:6829","nonce":1277310151}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6832","nonce":1277310151},{"type":"v1","addr":"172.21.15.87:6833","nonce":1277310151}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6830","nonce":1277310151},{"type":"v1","addr":"172.21.15.87:6831","nonce":1277310151}]},"public_addr":"172.21.15.87:6827/1277310151","cluster_addr":"172.21.15.87:6829/1277310151","heartbeat_back_addr":"172.21.15.87:6833/1277310151","heartbeat_front_addr":"172.21.15.87:6831/1277310151","state":["exists","up"]},{"osd":4,"uuid":"dcd6edd9-74b5-4ba9-9282-e9325c914835","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6800","nonce":2668368842},{"type":"v1","addr":"172.21.15.106:6801","nonce":2668368842}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6802","nonce":2668368842},{"type":"v1","addr":"172.21.15.106:6803","nonce":2668368842}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6806","nonce":2668368842},{"type":"v1","addr":"172.21.15.106:6807","nonce":2668368842}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6804","nonce":2668368842},{"type":"v1","addr":"172.21.15.106:6805","nonce":2668368842}]},"public_addr":"172.21.15.106:6801/2668368842","cluster_addr":"172.21.15.106:6803/2668368842","heartbeat_back_addr":"172.21.15.106:6807/2668368842","heartbeat_front_addr":"172.21.15.106:6805/2668368842","state":["exists","up"]},{"osd":5,"uuid":"4c205155-70f2-4d96-aebb-3d0e96ba5a95","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6808","nonce":2879909052},{"type":"v1","addr":"172.21.15.106:6809","nonce":2879909052}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6810","nonce":2879909052},{"type":"v1","addr":"172.21.15.106:6811","nonce":2879909052}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6814","nonce":2879909052},{"type":"v1","addr":"172.21.15.106:6815","nonce":2879909052}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6812","nonce":2879909052},{"type":"v1","addr":"172.21.15.106:6813","nonce":2879909052}]},"public_addr":"172.21.15.106:6809/2879909052","cluster_addr":"172.21.15.106:6811/2879909052","heartbeat_back_addr":"172.21.15.106:6815/2879909052","heartbeat_front_addr":"172.21.15.106:6813/2879909052","state":["exists","up"]},{"osd":6,"uuid":"391fb1af-ab61-4a21-8bf3-da6f0117756e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6816","nonce":125323095},{"type":"v1","addr":"172.21.15.106:6817","nonce":125323095}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6818","nonce":125323095},{"type":"v1","addr":"172.21.15.106:6819","nonce":125323095}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6822","nonce":125323095},{"type":"v1","addr":"172.21.15.106:6823","nonce":125323095}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6820","nonce":125323095},{"type":"v1","addr":"172.21.15.106:6821","nonce":125323095}]},"public_addr":"172.21.15.106:6817/125323095","cluster_addr":"172.21.15.106:6819/125323095","heartbeat_back_addr":"172.21.15.106:6823/125323095","heartbeat_front_addr":"172.21.15.106:6821/125323095","state":["exists","up"]},{"osd":7,"uuid":"780bffc8-5404-40eb-860f-26c6e0c773b6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6824","nonce":256892273},{"type":"v1","addr":"172.21.15.106:6825","nonce":256892273}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6826","nonce":256892273},{"type":"v1","addr":"172.21.15.106:6827","nonce":256892273}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6830","nonce":256892273},{"type":"v1","addr":"172.21.15.106:6831","nonce":256892273}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6828","nonce":256892273},{"type":"v1","addr":"172.21.15.106:6829","nonce":256892273}]},"public_addr":"172.21.15.106:6825/256892273","cluster_addr":"172.21.15.106:6827/256892273","heartbeat_back_addr":"172.21.15.106:6831/256892273","heartbeat_front_addr":"172.21.15.106:6829/256892273","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T15:26:37.310106+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T15:26:59.114255+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T15:27:22.084772+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T15:27:45.261604+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T15:28:08.046780+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T15:28:31.088909+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T15:28:53.265880+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T15:29:15.668954+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.87:0/2189995990":"2024-04-03T15:25:16.785348+0000","172.21.15.87:0/1859893958":"2024-04-03T15:25:16.785348+0000","172.21.15.87:6800/3965460628":"2024-04-03T15:25:16.785348+0000","172.21.15.87:6800/2380456996":"2024-04-03T15:24:54.083807+0000","172.21.15.87:0/2076084295":"2024-04-03T15:25:16.785348+0000","172.21.15.87:6801/2380456996":"2024-04-03T15:24:54.083807+0000","172.21.15.87:6801/3965460628":"2024-04-03T15:25:16.785348+0000","172.21.15.87:0/3045002547":"2024-04-03T15:24:54.083807+0000","172.21.15.87:0/2722718326":"2024-04-03T15:24:54.083807+0000","172.21.15.87:0/593333980":"2024-04-03T15:24:54.083807+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-02T15:29:23.738 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:23 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/1368163347' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T15:29:23.738 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:23 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/1368163347' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T15:29:23.760 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-02T15:27:25.144519+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-04-02T15:29:23.760 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph osd pool get .mgr pg_num 2024-04-02T15:29:23.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:23 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/1368163347' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T15:29:23.972 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:29:24.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:24 smithi106 ceph-mon[25843]: pgmap v177: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 124 KiB/s, 0 objects/s recovering 2024-04-02T15:29:24.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:24 smithi087 ceph-mon[24900]: pgmap v177: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 124 KiB/s, 0 objects/s recovering 2024-04-02T15:29:24.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:24 smithi087 ceph-mon[29707]: pgmap v177: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 124 KiB/s, 0 objects/s recovering 2024-04-02T15:29:25.187 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:24 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:29:24] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:29:25.589 INFO:teuthology.orchestra.run.smithi087.stdout:pg_num: 1 2024-04-02T15:29:26.226 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:25 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:26.226 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:25 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:26.226 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:25 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/4205260046' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-02T15:29:26.227 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:25 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:26.227 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:25 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:26.227 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:25 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/4205260046' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-02T15:29:26.251 INFO:tasks.cephadm:Adding prometheus.a on smithi106 2024-04-02T15:29:26.251 DEBUG:teuthology.orchestra.run.smithi106:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph orch apply prometheus '1;smithi106=a' 2024-04-02T15:29:26.279 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:25 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:26.279 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:25 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:26.279 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:25 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/4205260046' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-02T15:29:26.980 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:26 smithi087 ceph-mon[24900]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-02T15:29:26.980 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:26 smithi087 ceph-mon[29707]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-02T15:29:27.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:26 smithi106 ceph-mon[25843]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-02T15:29:28.025 INFO:teuthology.orchestra.run.smithi106.stdout:Scheduled prometheus update... 2024-04-02T15:29:28.603 DEBUG:teuthology.orchestra.run.smithi106:prometheus.a> sudo journalctl -f -n 0 -u ceph-0400842c-f105-11ee-b647-cb9ed24678a4@prometheus.a.service 2024-04-02T15:29:28.645 INFO:tasks.cephadm:Adding node-exporter.a on smithi087 2024-04-02T15:29:28.645 INFO:tasks.cephadm:Adding node-exporter.b on smithi106 2024-04-02T15:29:28.646 DEBUG:teuthology.orchestra.run.smithi106:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph orch apply node-exporter '2;smithi087=a;smithi106=b' 2024-04-02T15:29:29.097 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:28 smithi087 ceph-mon[29707]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-02T15:29:29.098 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:28 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:29.098 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:28 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:29:29.098 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:28 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:29:29.098 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:28 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:29:29.098 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:28 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:29.098 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:28 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-02T15:29:29.098 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:28 smithi087 ceph-mon[24900]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-02T15:29:29.098 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:28 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:29.098 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:28 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:29:29.098 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:28 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:29:29.099 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:28 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:29:29.099 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:28 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:29.099 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:28 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-02T15:29:29.183 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:28 smithi106 ceph-mon[25843]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-02T15:29:29.184 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:28 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:29.184 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:28 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:29:29.184 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:28 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:29:29.184 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:28 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:29:29.184 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:28 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' 2024-04-02T15:29:29.184 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:28 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-02T15:29:29.184 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:29:29 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ignoring --setuser ceph since I am not root 2024-04-02T15:29:29.184 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:29:29 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ignoring --setgroup ceph since I am not root 2024-04-02T15:29:29.184 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:29:29 smithi106 ceph-mgr[27389]: -- 172.21.15.106:0/2044759546 <== mon.2 v2:172.21.15.106:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x562d7ee18340 con 0x562d7edf1000 2024-04-02T15:29:29.370 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:29 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ignoring --setuser ceph since I am not root 2024-04-02T15:29:29.370 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:29 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ignoring --setgroup ceph since I am not root 2024-04-02T15:29:29.370 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:29 smithi087 ceph-mgr[25136]: -- 172.21.15.87:0/2599457691 <== mon.1 v2:172.21.15.87:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5633675a0340 con 0x563367579000 2024-04-02T15:29:29.836 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:29:29 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:29:29.570+0000 7f75933ba000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-02T15:29:29.836 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:29:29 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:29:29.662+0000 7f75933ba000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-02T15:29:29.870 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:29 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:29:29.577+0000 7ff9471c2000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-02T15:29:29.870 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:29 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:29:29.667+0000 7ff9471c2000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-02T15:29:30.091 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:29 smithi106 ceph-mon[25843]: from='client.24341 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi106=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:29:30.091 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:29 smithi106 ceph-mon[25843]: Saving service prometheus spec with placement smithi106=a;count:1 2024-04-02T15:29:30.091 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:29 smithi106 ceph-mon[25843]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-02T15:29:30.091 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:29 smithi106 ceph-mon[25843]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-02T15:29:30.254 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:29 smithi087 ceph-mon[29707]: from='client.24341 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi106=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:29:30.255 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:29 smithi087 ceph-mon[29707]: Saving service prometheus spec with placement smithi106=a;count:1 2024-04-02T15:29:30.255 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:29 smithi087 ceph-mon[29707]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-02T15:29:30.255 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:29 smithi087 ceph-mon[29707]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-02T15:29:30.255 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:29 smithi087 ceph-mon[24900]: from='client.24341 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi106=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:29:30.255 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:29 smithi087 ceph-mon[24900]: Saving service prometheus spec with placement smithi106=a;count:1 2024-04-02T15:29:30.255 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:29 smithi087 ceph-mon[24900]: from='mgr.14150 172.21.15.87:0/2314014090' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-02T15:29:30.255 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:29 smithi087 ceph-mon[24900]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-02T15:29:30.359 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:29:30 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:29:30.267+0000 7f75933ba000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-02T15:29:30.359 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:29:30 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:29:30.363+0000 7f75933ba000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-02T15:29:30.575 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:30 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:29:30.257+0000 7ff9471c2000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-02T15:29:30.576 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:30 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:29:30.346+0000 7ff9471c2000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-02T15:29:30.576 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:30 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:29:30.429+0000 7ff9471c2000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-02T15:29:30.576 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:30 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:29:30.578+0000 7ff9471c2000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-02T15:29:30.616 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:29:30 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:29:30.448+0000 7f75933ba000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-02T15:29:30.616 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:29:30 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:29:30.596+0000 7f75933ba000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-02T15:29:30.846 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:30 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:29:30.658+0000 7ff9471c2000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-02T15:29:30.846 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:30 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:29:30.767+0000 7ff9471c2000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-02T15:29:30.846 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:30 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:29:30.849+0000 7ff9471c2000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-02T15:29:31.024 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:29:30 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:29:30.677+0000 7f75933ba000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-02T15:29:31.024 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:29:30 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:29:30.784+0000 7f75933ba000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-02T15:29:31.024 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:29:30 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:29:30.866+0000 7f75933ba000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-02T15:29:31.025 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:29:31 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:29:31.028+0000 7f75933ba000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-02T15:29:31.120 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:31 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:29:31.011+0000 7ff9471c2000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-02T15:29:31.120 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:31 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:29:31.095+0000 7ff9471c2000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-02T15:29:31.366 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:29:31 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:29:31.112+0000 7f75933ba000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-02T15:29:32.370 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:32 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:29:32.116+0000 7ff9471c2000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-02T15:29:32.370 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:32 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:29:32.245+0000 7ff9471c2000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-02T15:29:32.433 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:29:32 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:29:32.120+0000 7f75933ba000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-02T15:29:32.433 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:29:32 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:29:32.248+0000 7f75933ba000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-02T15:29:32.433 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:29:32 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:29:32.437+0000 7f75933ba000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-02T15:29:32.866 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:29:32 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:29:32.522+0000 7f75933ba000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-02T15:29:32.866 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:29:32 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:29:32.654+0000 7f75933ba000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-02T15:29:32.870 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:32 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:29:32.436+0000 7ff9471c2000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-02T15:29:32.870 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:32 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:29:32.524+0000 7ff9471c2000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-02T15:29:32.870 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:32 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:29:32.659+0000 7ff9471c2000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-02T15:29:33.264 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:29:32 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:29:32.968+0000 7f75933ba000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-02T15:29:33.264 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:29:33 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:29:33.185+0000 7f75933ba000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-02T15:29:33.264 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:29:33 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:29:33.268+0000 7f75933ba000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-02T15:29:33.286 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:32 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:29:32.980+0000 7ff9471c2000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-02T15:29:33.286 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:33 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:29:33.202+0000 7ff9471c2000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-02T15:29:33.286 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:33 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:29:33.289+0000 7ff9471c2000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-02T15:29:33.616 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:29:33 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:29:33.435+0000 7f75933ba000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-02T15:29:33.620 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:33 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:29:33.463+0000 7ff9471c2000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-02T15:29:34.616 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:29:34 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:29:34.335+0000 7f75933ba000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-02T15:29:34.682 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:34 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:29:34.387+0000 7ff9471c2000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-02T15:29:34.682 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:34 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:29:34.685+0000 7ff9471c2000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-02T15:29:34.969 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:29:34 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:29:34.624+0000 7f75933ba000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-02T15:29:34.969 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:29:34 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:29:34.717+0000 7f75933ba000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-02T15:29:34.969 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:29:34 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:29:34.865+0000 7f75933ba000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-02T15:29:34.970 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:29:34 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: [02/Apr/2024:15:29:34] ENGINE Bus STARTING 2024-04-02T15:29:34.970 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:29:34 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: CherryPy Checker: 2024-04-02T15:29:34.970 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:29:34 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: The Application mounted at '' has an empty config. 2024-04-02T15:29:34.970 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:29:34 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: 2024-04-02T15:29:34.970 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:29:34 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: [02/Apr/2024:15:29:34] ENGINE Serving on http://:::9283 2024-04-02T15:29:34.970 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:29:34 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: [02/Apr/2024:15:29:34] ENGINE Bus STARTED 2024-04-02T15:29:35.120 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:34 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:29:34.782+0000 7ff9471c2000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-02T15:29:35.120 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:34 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:29:34.933+0000 7ff9471c2000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-02T15:29:35.300 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:35 smithi106 ceph-mon[25843]: Standby manager daemon x restarted 2024-04-02T15:29:35.301 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:35 smithi106 ceph-mon[25843]: Standby manager daemon x started 2024-04-02T15:29:35.301 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:35 smithi106 ceph-mon[25843]: from='mgr.? 172.21.15.106:0/2218496500' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-02T15:29:35.301 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:35 smithi106 ceph-mon[25843]: from='mgr.? 172.21.15.106:0/2218496500' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T15:29:35.301 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:35 smithi106 ceph-mon[25843]: from='mgr.? 172.21.15.106:0/2218496500' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-02T15:29:35.301 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:35 smithi106 ceph-mon[25843]: from='mgr.? 172.21.15.106:0/2218496500' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T15:29:35.301 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:35 smithi106 ceph-mon[25843]: Active manager daemon y restarted 2024-04-02T15:29:35.301 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:35 smithi106 ceph-mon[25843]: Activating manager daemon y 2024-04-02T15:29:35.301 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:35 smithi106 ceph-mon[25843]: osdmap e59: 8 total, 8 up, 8 in 2024-04-02T15:29:35.530 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:35 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: [02/Apr/2024:15:29:35] ENGINE Bus STARTING 2024-04-02T15:29:35.531 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:35 smithi087 ceph-mon[24900]: Standby manager daemon x restarted 2024-04-02T15:29:35.531 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:35 smithi087 ceph-mon[24900]: Standby manager daemon x started 2024-04-02T15:29:35.531 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:35 smithi087 ceph-mon[24900]: from='mgr.? 172.21.15.106:0/2218496500' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-02T15:29:35.531 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:35 smithi087 ceph-mon[24900]: from='mgr.? 172.21.15.106:0/2218496500' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T15:29:35.531 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:35 smithi087 ceph-mon[24900]: from='mgr.? 172.21.15.106:0/2218496500' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-02T15:29:35.531 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:35 smithi087 ceph-mon[24900]: from='mgr.? 172.21.15.106:0/2218496500' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T15:29:35.531 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:35 smithi087 ceph-mon[24900]: Active manager daemon y restarted 2024-04-02T15:29:35.531 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:35 smithi087 ceph-mon[24900]: Activating manager daemon y 2024-04-02T15:29:35.532 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:35 smithi087 ceph-mon[24900]: osdmap e59: 8 total, 8 up, 8 in 2024-04-02T15:29:35.532 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:35 smithi087 ceph-mon[29707]: Standby manager daemon x restarted 2024-04-02T15:29:35.532 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:35 smithi087 ceph-mon[29707]: Standby manager daemon x started 2024-04-02T15:29:35.532 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:35 smithi087 ceph-mon[29707]: from='mgr.? 172.21.15.106:0/2218496500' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-02T15:29:35.532 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:35 smithi087 ceph-mon[29707]: from='mgr.? 172.21.15.106:0/2218496500' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T15:29:35.532 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:35 smithi087 ceph-mon[29707]: from='mgr.? 172.21.15.106:0/2218496500' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-02T15:29:35.532 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:35 smithi087 ceph-mon[29707]: from='mgr.? 172.21.15.106:0/2218496500' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T15:29:35.532 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:35 smithi087 ceph-mon[29707]: Active manager daemon y restarted 2024-04-02T15:29:35.532 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:35 smithi087 ceph-mon[29707]: Activating manager daemon y 2024-04-02T15:29:35.532 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:35 smithi087 ceph-mon[29707]: osdmap e59: 8 total, 8 up, 8 in 2024-04-02T15:29:35.782 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:35 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: CherryPy Checker: 2024-04-02T15:29:35.782 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:35 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: The Application mounted at '' has an empty config. 2024-04-02T15:29:35.782 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:35 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:29:35.782 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:35 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: [02/Apr/2024:15:29:35] ENGINE Serving on http://:::9283 2024-04-02T15:29:35.782 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:35 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: [02/Apr/2024:15:29:35] ENGINE Bus STARTED 2024-04-02T15:29:35.782 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:35 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: [02/Apr/2024:15:29:35] ENGINE Bus STARTING 2024-04-02T15:29:35.782 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:35 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: [02/Apr/2024:15:29:35] ENGINE Serving on https://172.21.15.87:7150 2024-04-02T15:29:35.782 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:35 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: [02/Apr/2024:15:29:35] ENGINE Bus STARTED 2024-04-02T15:29:36.226 INFO:teuthology.orchestra.run.smithi106.stdout:Scheduled node-exporter update... 2024-04-02T15:29:36.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:36 smithi106 ceph-mon[25843]: mgrmap e17: y(active, starting, since 0.274377s), standbys: x 2024-04-02T15:29:36.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T15:29:36.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:29:36.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:29:36.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T15:29:36.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-02T15:29:36.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T15:29:36.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T15:29:36.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T15:29:36.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T15:29:36.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T15:29:36.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T15:29:36.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T15:29:36.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:29:36.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T15:29:36.368 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T15:29:36.368 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T15:29:36.368 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:36 smithi106 ceph-mon[25843]: Manager daemon y is now available 2024-04-02T15:29:36.368 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:36.368 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:29:36.368 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:29:36.368 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:29:36.368 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:29:36.368 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi087", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:29:36.368 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi106", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:29:36.368 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd='[{"prefix": "config rm", "who": "osd/host:smithi087", "name": "osd_memory_target"}]': finished 2024-04-02T15:29:36.368 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:36 smithi106 ceph-mon[25843]: Updating smithi087:/etc/ceph/ceph.conf 2024-04-02T15:29:36.368 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:36.368 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd='[{"prefix": "config rm", "who": "osd/host:smithi106", "name": "osd_memory_target"}]': finished 2024-04-02T15:29:36.368 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:36 smithi106 ceph-mon[25843]: Updating smithi106:/etc/ceph/ceph.conf 2024-04-02T15:29:36.369 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:29:36.369 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:29:36.369 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:36 smithi106 ceph-mon[25843]: [02/Apr/2024:15:29:35] ENGINE Bus STARTING 2024-04-02T15:29:36.369 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:36 smithi106 ceph-mon[25843]: [02/Apr/2024:15:29:35] ENGINE Serving on https://172.21.15.87:7150 2024-04-02T15:29:36.369 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:36 smithi106 ceph-mon[25843]: [02/Apr/2024:15:29:35] ENGINE Bus STARTED 2024-04-02T15:29:36.369 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:36.369 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:36 smithi106 ceph-mon[25843]: Updating smithi106:/etc/ceph/ceph.client.admin.keyring 2024-04-02T15:29:36.369 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:36 smithi106 ceph-mon[25843]: Updating smithi087:/etc/ceph/ceph.client.admin.keyring 2024-04-02T15:29:36.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[24900]: mgrmap e17: y(active, starting, since 0.274377s), standbys: x 2024-04-02T15:29:36.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T15:29:36.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:29:36.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:29:36.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T15:29:36.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-02T15:29:36.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T15:29:36.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T15:29:36.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T15:29:36.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T15:29:36.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T15:29:36.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T15:29:36.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T15:29:36.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:29:36.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T15:29:36.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T15:29:36.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T15:29:36.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[24900]: Manager daemon y is now available 2024-04-02T15:29:36.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:36.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:29:36.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:29:36.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:29:36.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:29:36.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi087", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:29:36.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi106", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:29:36.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd='[{"prefix": "config rm", "who": "osd/host:smithi087", "name": "osd_memory_target"}]': finished 2024-04-02T15:29:36.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[24900]: Updating smithi087:/etc/ceph/ceph.conf 2024-04-02T15:29:36.372 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[29707]: mgrmap e17: y(active, starting, since 0.274377s), standbys: x 2024-04-02T15:29:36.372 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T15:29:36.372 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:29:36.372 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:29:36.372 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T15:29:36.373 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-02T15:29:36.373 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T15:29:36.373 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T15:29:36.373 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T15:29:36.373 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T15:29:36.373 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T15:29:36.373 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T15:29:36.373 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T15:29:36.373 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:29:36.373 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T15:29:36.373 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T15:29:36.373 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T15:29:36.373 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[29707]: Manager daemon y is now available 2024-04-02T15:29:36.373 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:36.373 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:29:36.374 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:29:36.374 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:29:36.374 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:29:36.374 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi087", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:29:36.374 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi106", "name": "osd_memory_target"}]: dispatch 2024-04-02T15:29:36.374 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd='[{"prefix": "config rm", "who": "osd/host:smithi087", "name": "osd_memory_target"}]': finished 2024-04-02T15:29:36.374 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[29707]: Updating smithi087:/etc/ceph/ceph.conf 2024-04-02T15:29:36.374 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:36.374 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd='[{"prefix": "config rm", "who": "osd/host:smithi106", "name": "osd_memory_target"}]': finished 2024-04-02T15:29:36.374 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[24900]: Updating smithi106:/etc/ceph/ceph.conf 2024-04-02T15:29:36.374 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:29:36.375 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:29:36.375 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[24900]: [02/Apr/2024:15:29:35] ENGINE Bus STARTING 2024-04-02T15:29:36.375 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[24900]: [02/Apr/2024:15:29:35] ENGINE Serving on https://172.21.15.87:7150 2024-04-02T15:29:36.375 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[24900]: [02/Apr/2024:15:29:35] ENGINE Bus STARTED 2024-04-02T15:29:36.375 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:36.375 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[24900]: Updating smithi106:/etc/ceph/ceph.client.admin.keyring 2024-04-02T15:29:36.375 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[24900]: Updating smithi087:/etc/ceph/ceph.client.admin.keyring 2024-04-02T15:29:36.375 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:36.375 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd='[{"prefix": "config rm", "who": "osd/host:smithi106", "name": "osd_memory_target"}]': finished 2024-04-02T15:29:36.375 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[29707]: Updating smithi106:/etc/ceph/ceph.conf 2024-04-02T15:29:36.375 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:29:36.375 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:29:36.375 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[29707]: [02/Apr/2024:15:29:35] ENGINE Bus STARTING 2024-04-02T15:29:36.375 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[29707]: [02/Apr/2024:15:29:35] ENGINE Serving on https://172.21.15.87:7150 2024-04-02T15:29:36.376 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[29707]: [02/Apr/2024:15:29:35] ENGINE Bus STARTED 2024-04-02T15:29:36.376 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:36.376 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[29707]: Updating smithi106:/etc/ceph/ceph.client.admin.keyring 2024-04-02T15:29:36.376 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:36 smithi087 ceph-mon[29707]: Updating smithi087:/etc/ceph/ceph.client.admin.keyring 2024-04-02T15:29:36.812 DEBUG:teuthology.orchestra.run.smithi087:node-exporter.a> sudo journalctl -f -n 0 -u ceph-0400842c-f105-11ee-b647-cb9ed24678a4@node-exporter.a.service 2024-04-02T15:29:36.814 DEBUG:teuthology.orchestra.run.smithi106:node-exporter.b> sudo journalctl -f -n 0 -u ceph-0400842c-f105-11ee-b647-cb9ed24678a4@node-exporter.b.service 2024-04-02T15:29:36.817 INFO:tasks.cephadm:Adding alertmanager.a on smithi087 2024-04-02T15:29:36.817 DEBUG:teuthology.orchestra.run.smithi106:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph orch apply alertmanager '1;smithi087=a' 2024-04-02T15:29:37.489 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:37 smithi106 ceph-mon[25843]: mgrmap e18: y(active, since 1.2798s), standbys: x 2024-04-02T15:29:37.489 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:37 smithi106 ceph-mon[25843]: from='client.24359 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi087=a;smithi106=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:29:37.490 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:37 smithi106 ceph-mon[25843]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:37.490 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:37 smithi106 ceph-mon[25843]: Saving service node-exporter spec with placement smithi087=a;smithi106=b;count:2 2024-04-02T15:29:37.490 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:37 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:37.490 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:37 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:37.490 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:37 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:37.490 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:37 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:37.490 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:37 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-02T15:29:37.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:37 smithi087 ceph-mon[24900]: mgrmap e18: y(active, since 1.2798s), standbys: x 2024-04-02T15:29:37.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:37 smithi087 ceph-mon[24900]: from='client.24359 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi087=a;smithi106=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:29:37.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:37 smithi087 ceph-mon[24900]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:37.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:37 smithi087 ceph-mon[24900]: Saving service node-exporter spec with placement smithi087=a;smithi106=b;count:2 2024-04-02T15:29:37.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:37 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:37.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:37 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:37.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:37 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:37.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:37 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:37.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:37 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-02T15:29:37.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:37 smithi087 ceph-mon[29707]: mgrmap e18: y(active, since 1.2798s), standbys: x 2024-04-02T15:29:37.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:37 smithi087 ceph-mon[29707]: from='client.24359 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi087=a;smithi106=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:29:37.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:37 smithi087 ceph-mon[29707]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:37.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:37 smithi087 ceph-mon[29707]: Saving service node-exporter spec with placement smithi087=a;smithi106=b;count:2 2024-04-02T15:29:37.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:37 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:37.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:37 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:37.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:37 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:37.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:37 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:37.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:37 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-02T15:29:38.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:38 smithi106 ceph-mon[25843]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:38.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:38 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-02T15:29:38.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:38 smithi106 ceph-mon[25843]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-02T15:29:38.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:38 smithi106 ceph-mon[25843]: Deploying daemon prometheus.a on smithi106 2024-04-02T15:29:38.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:38 smithi087 ceph-mon[24900]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:38.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:38 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-02T15:29:38.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:38 smithi087 ceph-mon[24900]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-02T15:29:38.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:38 smithi087 ceph-mon[24900]: Deploying daemon prometheus.a on smithi106 2024-04-02T15:29:38.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:38 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:29:38] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:29:38.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:38 smithi087 ceph-mon[29707]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:38.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:38 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-02T15:29:38.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:38 smithi087 ceph-mon[29707]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-02T15:29:38.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:38 smithi087 ceph-mon[29707]: Deploying daemon prometheus.a on smithi106 2024-04-02T15:29:38.825 INFO:teuthology.orchestra.run.smithi106.stdout:Scheduled alertmanager update... 2024-04-02T15:29:39.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:39 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:39.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:39 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:39.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:39 smithi106 ceph-mon[25843]: from='client.24383 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi087=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:29:39.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:39 smithi106 ceph-mon[25843]: Saving service alertmanager spec with placement smithi087=a;count:1 2024-04-02T15:29:39.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:39 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:39.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:39 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:39.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:39 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:39.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:39 smithi087 ceph-mon[24900]: from='client.24383 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi087=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:29:39.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:39 smithi087 ceph-mon[24900]: Saving service alertmanager spec with placement smithi087=a;count:1 2024-04-02T15:29:39.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:39 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:39.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:39 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:39.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:39 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:39.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:39 smithi087 ceph-mon[29707]: from='client.24383 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi087=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:29:39.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:39 smithi087 ceph-mon[29707]: Saving service alertmanager spec with placement smithi087=a;count:1 2024-04-02T15:29:39.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:39 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:39.642 DEBUG:teuthology.orchestra.run.smithi087:alertmanager.a> sudo journalctl -f -n 0 -u ceph-0400842c-f105-11ee-b647-cb9ed24678a4@alertmanager.a.service 2024-04-02T15:29:39.645 INFO:tasks.cephadm:Adding grafana.a on smithi106 2024-04-02T15:29:39.645 DEBUG:teuthology.orchestra.run.smithi106:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph orch apply grafana '1;smithi106=a' 2024-04-02T15:29:40.458 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:40 smithi106 ceph-mon[25843]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:40.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:40 smithi087 ceph-mon[24900]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:40.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:40 smithi087 ceph-mon[29707]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:42.500 INFO:teuthology.orchestra.run.smithi106.stdout:Scheduled grafana update... 2024-04-02T15:29:42.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:42 smithi106 ceph-mon[25843]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:42.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:42 smithi087 ceph-mon[24900]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:42.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:42 smithi087 ceph-mon[29707]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:43.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:43 smithi087 ceph-mon[29707]: from='client.24389 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi106=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:29:43.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:43 smithi087 ceph-mon[29707]: Saving service grafana spec with placement smithi106=a;count:1 2024-04-02T15:29:43.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:43 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:43.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:43 smithi087 ceph-mon[24900]: from='client.24389 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi106=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:29:43.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:43 smithi087 ceph-mon[24900]: Saving service grafana spec with placement smithi106=a;count:1 2024-04-02T15:29:43.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:43 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:43.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:43 smithi106 ceph-mon[25843]: from='client.24389 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi106=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:29:43.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:43 smithi106 ceph-mon[25843]: Saving service grafana spec with placement smithi106=a;count:1 2024-04-02T15:29:43.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:43 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:44.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:44 smithi106 ceph-mon[25843]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:44.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:44 smithi087 ceph-mon[24900]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:44.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:44 smithi087 ceph-mon[29707]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:45.370 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:45 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:29:45] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:29:45.996 DEBUG:teuthology.orchestra.run.smithi106:grafana.a> sudo journalctl -f -n 0 -u ceph-0400842c-f105-11ee-b647-cb9ed24678a4@grafana.a.service 2024-04-02T15:29:45.999 INFO:tasks.cephadm:Setting up client nodes... 2024-04-02T15:29:46.002 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-02T15:29:46.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:45 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:46.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:45 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:46.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:45 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:46.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:45 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:46.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:45 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:46.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:45 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:47.320 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:47 smithi087 ceph-mon[29707]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:47.320 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:47 smithi087 ceph-mon[24900]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:47.322 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:47 smithi106 ceph-mon[25843]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:48.539 INFO:teuthology.orchestra.run.smithi087.stdout:[client.0] 2024-04-02T15:29:48.540 INFO:teuthology.orchestra.run.smithi087.stdout: key = AQBsJAxmi9Y8HRAAX0T/Uo3V7ZvDnxZoS8Ywaw== 2024-04-02T15:29:49.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:49 smithi087 ceph-mon[29707]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:49.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:49 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/1577788677' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T15:29:49.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:49 smithi087 ceph-mon[29707]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T15:29:49.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:49 smithi087 ceph-mon[29707]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-02T15:29:49.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:49 smithi087 ceph-mon[24900]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:49.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:49 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/1577788677' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T15:29:49.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:49 smithi087 ceph-mon[24900]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T15:29:49.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:49 smithi087 ceph-mon[24900]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-02T15:29:49.232 DEBUG:teuthology.orchestra.run.smithi087:> set -ex 2024-04-02T15:29:49.233 DEBUG:teuthology.orchestra.run.smithi087:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-04-02T15:29:49.233 DEBUG:teuthology.orchestra.run.smithi087:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-04-02T15:29:49.276 DEBUG:teuthology.orchestra.run.smithi106:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-02T15:29:49.307 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:49 smithi106 ceph-mon[25843]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:49.307 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:49 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/1577788677' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T15:29:49.307 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:49 smithi106 ceph-mon[25843]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T15:29:49.307 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:49 smithi106 ceph-mon[25843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-02T15:29:50.594 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:50 smithi087 ceph-mon[24900]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:50.595 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:50 smithi087 ceph-mon[29707]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:50.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:50 smithi106 ceph-mon[25843]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:51.474 INFO:teuthology.orchestra.run.smithi106.stdout:[client.1] 2024-04-02T15:29:51.475 INFO:teuthology.orchestra.run.smithi106.stdout: key = AQBvJAxm0y5JHBAAOyzb6Bnjo6VMAA1Wh2UHVQ== 2024-04-02T15:29:52.298 DEBUG:teuthology.orchestra.run.smithi106:> set -ex 2024-04-02T15:29:52.298 DEBUG:teuthology.orchestra.run.smithi106:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-04-02T15:29:52.298 DEBUG:teuthology.orchestra.run.smithi106:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-04-02T15:29:52.381 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-02T15:29:52.381 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-02T15:29:52.381 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph mgr dump --format=json 2024-04-02T15:29:52.509 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:52 smithi106 ceph-mon[25843]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:52.509 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:52 smithi106 ceph-mon[25843]: from='client.? 172.21.15.106:0/2942587957' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T15:29:52.509 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:52 smithi106 ceph-mon[25843]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T15:29:52.510 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:52 smithi106 ceph-mon[25843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-02T15:29:52.510 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:52 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:52.510 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:52 smithi106 ceph-mon[25843]: Deploying daemon node-exporter.a on smithi087 2024-04-02T15:29:52.540 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:52 smithi087 ceph-mon[29707]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:52.540 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:52 smithi087 ceph-mon[29707]: from='client.? 172.21.15.106:0/2942587957' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T15:29:52.540 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:52 smithi087 ceph-mon[29707]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T15:29:52.540 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:52 smithi087 ceph-mon[29707]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-02T15:29:52.540 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:52 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:52.540 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:52 smithi087 ceph-mon[29707]: Deploying daemon node-exporter.a on smithi087 2024-04-02T15:29:52.540 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:52 smithi087 ceph-mon[24900]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:52.541 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:52 smithi087 ceph-mon[24900]: from='client.? 172.21.15.106:0/2942587957' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T15:29:52.541 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:52 smithi087 ceph-mon[24900]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T15:29:52.541 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:52 smithi087 ceph-mon[24900]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-02T15:29:52.541 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:52 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:52.541 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:52 smithi087 ceph-mon[24900]: Deploying daemon node-exporter.a on smithi087 2024-04-02T15:29:52.630 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:29:53.693 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:53 smithi087 bash[45635]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2024-04-02T15:29:53.992 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:53 smithi087 bash[45635]: Getting image source signatures 2024-04-02T15:29:53.992 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:53 smithi087 bash[45635]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2024-04-02T15:29:53.992 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:53 smithi087 bash[45635]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2024-04-02T15:29:53.992 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:53 smithi087 bash[45635]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2024-04-02T15:29:54.519 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-02T15:29:54.520 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:54 smithi087 ceph-mon[24900]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:54.520 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:54 smithi087 ceph-mon[29707]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:54.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:54 smithi106 ceph-mon[25843]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:55.436 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:55 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/360194962' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-02T15:29:55.436 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:55 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:55.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:55 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/360194962' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-02T15:29:55.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:55 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:55.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:55 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/360194962' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-02T15:29:55.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:55 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:55.843 INFO:teuthology.orchestra.run.smithi087.stdout:{"epoch":19,"active_gid":14448,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6800","nonce":1510011858},{"type":"v1","addr":"172.21.15.87:6801","nonce":1510011858}]},"active_addr":"172.21.15.87:6801/1510011858","active_change":"2024-04-02T15:29:34.935451+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":24347,"name":"x","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.87:8443/","prometheus":"http://172.21.15.87:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":59,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.87:0","nonce":4097148152}]},{"addrvec":[{"type":"v2","addr":"172.21.15.87:0","nonce":3401304952}]},{"addrvec":[{"type":"v2","addr":"172.21.15.87:0","nonce":3235897753}]},{"addrvec":[{"type":"v2","addr":"172.21.15.87:0","nonce":3974135677}]}]}} 2024-04-02T15:29:55.846 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-02T15:29:55.846 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-02T15:29:55.847 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-02T15:29:56.059 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:29:56.106 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:55 smithi087 bash[45635]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-04-02T15:29:56.106 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 bash[45635]: Writing manifest to image destination 2024-04-02T15:29:56.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-mon[29707]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:56.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-mon[24900]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:56.592 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:56 smithi106 ceph-mon[25843]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:56.744 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 podman[45635]: 2024-04-02 15:29:56.417170654 +0000 UTC m=+3.423306941 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-04-02T15:29:56.744 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 podman[45635]: 2024-04-02 15:29:56.626588255 +0000 UTC m=+3.632724573 container create feba1ed50125939c5960bc947f2a0933a74a9217bca82c963ab55bcc5e98798f (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-02T15:29:57.121 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 podman[45635]: 2024-04-02 15:29:56.851553892 +0000 UTC m=+3.857690171 container init feba1ed50125939c5960bc947f2a0933a74a9217bca82c963ab55bcc5e98798f (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-02T15:29:57.121 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 podman[45635]: 2024-04-02 15:29:56.854129211 +0000 UTC m=+3.860265487 container start feba1ed50125939c5960bc947f2a0933a74a9217bca82c963ab55bcc5e98798f (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-02T15:29:57.121 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.858Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-04-02T15:29:57.121 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.858Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-04-02T15:29:57.121 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.858Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-04-02T15:29:57.121 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.858Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-02T15:29:57.121 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.858Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-04-02T15:29:57.121 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=arp 2024-04-02T15:29:57.121 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=bcache 2024-04-02T15:29:57.121 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=bonding 2024-04-02T15:29:57.121 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=btrfs 2024-04-02T15:29:57.122 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=conntrack 2024-04-02T15:29:57.122 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=cpu 2024-04-02T15:29:57.122 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-04-02T15:29:57.122 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=diskstats 2024-04-02T15:29:57.122 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=dmi 2024-04-02T15:29:57.122 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=edac 2024-04-02T15:29:57.122 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=entropy 2024-04-02T15:29:57.123 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-04-02T15:29:57.123 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=filefd 2024-04-02T15:29:57.123 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=filesystem 2024-04-02T15:29:57.123 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=hwmon 2024-04-02T15:29:57.123 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=infiniband 2024-04-02T15:29:57.123 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=ipvs 2024-04-02T15:29:57.123 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=loadavg 2024-04-02T15:29:57.123 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=mdadm 2024-04-02T15:29:57.123 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=meminfo 2024-04-02T15:29:57.123 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=netclass 2024-04-02T15:29:57.123 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=netdev 2024-04-02T15:29:57.123 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=netstat 2024-04-02T15:29:57.123 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=nfs 2024-04-02T15:29:57.123 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=nfsd 2024-04-02T15:29:57.123 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=nvme 2024-04-02T15:29:57.124 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=os 2024-04-02T15:29:57.124 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-04-02T15:29:57.124 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=pressure 2024-04-02T15:29:57.124 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=rapl 2024-04-02T15:29:57.124 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=schedstat 2024-04-02T15:29:57.124 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=sockstat 2024-04-02T15:29:57.124 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=softnet 2024-04-02T15:29:57.124 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=stat 2024-04-02T15:29:57.124 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=tapestats 2024-04-02T15:29:57.124 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=textfile 2024-04-02T15:29:57.124 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-04-02T15:29:57.124 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=time 2024-04-02T15:29:57.124 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-04-02T15:29:57.124 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=uname 2024-04-02T15:29:57.124 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=vmstat 2024-04-02T15:29:57.125 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=xfs 2024-04-02T15:29:57.125 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:115 level=info collector=zfs 2024-04-02T15:29:57.125 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-04-02T15:29:57.125 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a[45826]: ts=2024-04-02T15:29:56.859Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-04-02T15:29:57.125 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 bash[45635]: feba1ed50125939c5960bc947f2a0933a74a9217bca82c963ab55bcc5e98798f 2024-04-02T15:29:57.125 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:29:56 smithi087 systemd[1]: Started Ceph node-exporter.a for 0400842c-f105-11ee-b647-cb9ed24678a4. 2024-04-02T15:29:57.490 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:57 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:29:57] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:29:57.490 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:29:57 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:29:57] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:29:58.009 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:29:57 smithi106 systemd[1]: Starting Ceph node-exporter.b for 0400842c-f105-11ee-b647-cb9ed24678a4... 2024-04-02T15:29:58.159 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:58 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:58.159 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:58 smithi087 ceph-mon[29707]: Deploying daemon node-exporter.b on smithi106 2024-04-02T15:29:58.159 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:58 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:58.159 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:58 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:58.160 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:58 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:58.160 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:58 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:58.160 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:58 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:58.160 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:58 smithi087 ceph-mon[24900]: Deploying daemon node-exporter.b on smithi106 2024-04-02T15:29:58.160 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:58 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:58.160 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:58 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:58.160 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:58 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:58.160 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:58 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:58.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:58 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:58.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:58 smithi106 ceph-mon[25843]: Deploying daemon node-exporter.b on smithi106 2024-04-02T15:29:58.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:58 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:58.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:58 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:58.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:58 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:58.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:58 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:29:58.367 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:29:58 smithi106 bash[42813]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2024-04-02T15:29:58.546 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-02T15:29:58.546 INFO:teuthology.orchestra.run.smithi087.stdout:{"epoch":59,"fsid":"0400842c-f105-11ee-b647-cb9ed24678a4","created":"2024-04-02T15:24:24.701153+0000","modified":"2024-04-02T15:29:34.935122+0000","last_up_change":"2024-04-02T15:29:17.597118+0000","last_in_change":"2024-04-02T15:28:59.589376+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-02T15:27:25.144519+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"13d89f7c-4e33-4521-a94a-ff0658e7dccf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6802","nonce":2380439245},{"type":"v1","addr":"172.21.15.87:6803","nonce":2380439245}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6804","nonce":2380439245},{"type":"v1","addr":"172.21.15.87:6805","nonce":2380439245}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6808","nonce":2380439245},{"type":"v1","addr":"172.21.15.87:6809","nonce":2380439245}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6806","nonce":2380439245},{"type":"v1","addr":"172.21.15.87:6807","nonce":2380439245}]},"public_addr":"172.21.15.87:6803/2380439245","cluster_addr":"172.21.15.87:6805/2380439245","heartbeat_back_addr":"172.21.15.87:6809/2380439245","heartbeat_front_addr":"172.21.15.87:6807/2380439245","state":["exists","up"]},{"osd":1,"uuid":"f8e09079-dd20-411b-a160-5d6906d1d7ed","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6810","nonce":1909345406},{"type":"v1","addr":"172.21.15.87:6811","nonce":1909345406}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6812","nonce":1909345406},{"type":"v1","addr":"172.21.15.87:6813","nonce":1909345406}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6816","nonce":1909345406},{"type":"v1","addr":"172.21.15.87:6817","nonce":1909345406}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6814","nonce":1909345406},{"type":"v1","addr":"172.21.15.87:6815","nonce":1909345406}]},"public_addr":"172.21.15.87:6811/1909345406","cluster_addr":"172.21.15.87:6813/1909345406","heartbeat_back_addr":"172.21.15.87:6817/1909345406","heartbeat_front_addr":"172.21.15.87:6815/1909345406","state":["exists","up"]},{"osd":2,"uuid":"55868ded-a2a8-48c7-8fb2-d452831755d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6818","nonce":2886422817},{"type":"v1","addr":"172.21.15.87:6819","nonce":2886422817}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6820","nonce":2886422817},{"type":"v1","addr":"172.21.15.87:6821","nonce":2886422817}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6824","nonce":2886422817},{"type":"v1","addr":"172.21.15.87:6825","nonce":2886422817}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6822","nonce":2886422817},{"type":"v1","addr":"172.21.15.87:6823","nonce":2886422817}]},"public_addr":"172.21.15.87:6819/2886422817","cluster_addr":"172.21.15.87:6821/2886422817","heartbeat_back_addr":"172.21.15.87:6825/2886422817","heartbeat_front_addr":"172.21.15.87:6823/2886422817","state":["exists","up"]},{"osd":3,"uuid":"fba679e6-6a92-4943-bbc9-3c446311dfc5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6826","nonce":1277310151},{"type":"v1","addr":"172.21.15.87:6827","nonce":1277310151}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6828","nonce":1277310151},{"type":"v1","addr":"172.21.15.87:6829","nonce":1277310151}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6832","nonce":1277310151},{"type":"v1","addr":"172.21.15.87:6833","nonce":1277310151}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6830","nonce":1277310151},{"type":"v1","addr":"172.21.15.87:6831","nonce":1277310151}]},"public_addr":"172.21.15.87:6827/1277310151","cluster_addr":"172.21.15.87:6829/1277310151","heartbeat_back_addr":"172.21.15.87:6833/1277310151","heartbeat_front_addr":"172.21.15.87:6831/1277310151","state":["exists","up"]},{"osd":4,"uuid":"dcd6edd9-74b5-4ba9-9282-e9325c914835","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6800","nonce":2668368842},{"type":"v1","addr":"172.21.15.106:6801","nonce":2668368842}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6802","nonce":2668368842},{"type":"v1","addr":"172.21.15.106:6803","nonce":2668368842}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6806","nonce":2668368842},{"type":"v1","addr":"172.21.15.106:6807","nonce":2668368842}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6804","nonce":2668368842},{"type":"v1","addr":"172.21.15.106:6805","nonce":2668368842}]},"public_addr":"172.21.15.106:6801/2668368842","cluster_addr":"172.21.15.106:6803/2668368842","heartbeat_back_addr":"172.21.15.106:6807/2668368842","heartbeat_front_addr":"172.21.15.106:6805/2668368842","state":["exists","up"]},{"osd":5,"uuid":"4c205155-70f2-4d96-aebb-3d0e96ba5a95","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6808","nonce":2879909052},{"type":"v1","addr":"172.21.15.106:6809","nonce":2879909052}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6810","nonce":2879909052},{"type":"v1","addr":"172.21.15.106:6811","nonce":2879909052}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6814","nonce":2879909052},{"type":"v1","addr":"172.21.15.106:6815","nonce":2879909052}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6812","nonce":2879909052},{"type":"v1","addr":"172.21.15.106:6813","nonce":2879909052}]},"public_addr":"172.21.15.106:6809/2879909052","cluster_addr":"172.21.15.106:6811/2879909052","heartbeat_back_addr":"172.21.15.106:6815/2879909052","heartbeat_front_addr":"172.21.15.106:6813/2879909052","state":["exists","up"]},{"osd":6,"uuid":"391fb1af-ab61-4a21-8bf3-da6f0117756e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6816","nonce":125323095},{"type":"v1","addr":"172.21.15.106:6817","nonce":125323095}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6818","nonce":125323095},{"type":"v1","addr":"172.21.15.106:6819","nonce":125323095}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6822","nonce":125323095},{"type":"v1","addr":"172.21.15.106:6823","nonce":125323095}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6820","nonce":125323095},{"type":"v1","addr":"172.21.15.106:6821","nonce":125323095}]},"public_addr":"172.21.15.106:6817/125323095","cluster_addr":"172.21.15.106:6819/125323095","heartbeat_back_addr":"172.21.15.106:6823/125323095","heartbeat_front_addr":"172.21.15.106:6821/125323095","state":["exists","up"]},{"osd":7,"uuid":"780bffc8-5404-40eb-860f-26c6e0c773b6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6824","nonce":256892273},{"type":"v1","addr":"172.21.15.106:6825","nonce":256892273}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6826","nonce":256892273},{"type":"v1","addr":"172.21.15.106:6827","nonce":256892273}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6830","nonce":256892273},{"type":"v1","addr":"172.21.15.106:6831","nonce":256892273}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6828","nonce":256892273},{"type":"v1","addr":"172.21.15.106:6829","nonce":256892273}]},"public_addr":"172.21.15.106:6825/256892273","cluster_addr":"172.21.15.106:6827/256892273","heartbeat_back_addr":"172.21.15.106:6831/256892273","heartbeat_front_addr":"172.21.15.106:6829/256892273","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T15:26:37.310106+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T15:26:59.114255+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T15:27:22.084772+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T15:27:45.261604+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T15:28:08.046780+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T15:28:31.088909+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T15:28:53.265880+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T15:29:15.668954+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.87:0/3267436387":"2024-04-03T15:29:34.935088+0000","172.21.15.87:6800/2108248912":"2024-04-03T15:29:34.935088+0000","172.21.15.87:0/2189995990":"2024-04-03T15:25:16.785348+0000","172.21.15.87:0/1859893958":"2024-04-03T15:25:16.785348+0000","172.21.15.87:6800/3965460628":"2024-04-03T15:25:16.785348+0000","172.21.15.87:0/4051190714":"2024-04-03T15:29:34.935088+0000","172.21.15.87:6800/2380456996":"2024-04-03T15:24:54.083807+0000","172.21.15.87:0/2150387812":"2024-04-03T15:29:34.935088+0000","172.21.15.87:0/2076084295":"2024-04-03T15:25:16.785348+0000","172.21.15.87:6801/2380456996":"2024-04-03T15:24:54.083807+0000","172.21.15.87:6801/3965460628":"2024-04-03T15:25:16.785348+0000","172.21.15.87:0/3045002547":"2024-04-03T15:24:54.083807+0000","172.21.15.87:0/2722718326":"2024-04-03T15:24:54.083807+0000","172.21.15.87:6801/2108248912":"2024-04-03T15:29:34.935088+0000","172.21.15.87:0/3328279098":"2024-04-03T15:29:34.935088+0000","172.21.15.87:0/593333980":"2024-04-03T15:24:54.083807+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-02T15:29:58.866 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:29:58 smithi106 bash[42813]: Getting image source signatures 2024-04-02T15:29:58.866 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:29:58 smithi106 bash[42813]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2024-04-02T15:29:58.866 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:29:58 smithi106 bash[42813]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2024-04-02T15:29:58.866 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:29:58 smithi106 bash[42813]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2024-04-02T15:29:59.151 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-02T15:29:59.151 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-02T15:29:59.177 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:59 smithi087 ceph-mon[29707]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:59.177 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:29:59 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/1085096704' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T15:29:59.178 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:59 smithi087 ceph-mon[24900]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:59.178 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:29:59 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/1085096704' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T15:29:59.365 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:29:59.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:59 smithi106 ceph-mon[25843]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:29:59.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:29:59 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/1085096704' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T15:30:00.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:00 smithi087 ceph-mon[29707]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:00.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:00 smithi087 ceph-mon[29707]: overall HEALTH_OK 2024-04-02T15:30:00.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:00 smithi087 ceph-mon[24900]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:00.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:00 smithi087 ceph-mon[24900]: overall HEALTH_OK 2024-04-02T15:30:00.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-mon[25843]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:00.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-mon[25843]: overall HEALTH_OK 2024-04-02T15:30:00.616 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 bash[42813]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-04-02T15:30:00.616 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 bash[42813]: Writing manifest to image destination 2024-04-02T15:30:01.022 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 podman[42813]: 2024-04-02 15:30:00.608252471 +0000 UTC m=+2.628476440 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-04-02T15:30:01.023 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 podman[42813]: 2024-04-02 15:30:00.715167225 +0000 UTC m=+2.735391237 container create fb10fb2b50d2b06c18ba92c000ab7cf8e50aa9b7704fa418f03444e9c823d934 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-02T15:30:01.023 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 podman[42813]: 2024-04-02 15:30:00.925875171 +0000 UTC m=+2.946099141 container init fb10fb2b50d2b06c18ba92c000ab7cf8e50aa9b7704fa418f03444e9c823d934 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-02T15:30:01.023 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 podman[42813]: 2024-04-02 15:30:00.928640729 +0000 UTC m=+2.948864707 container start fb10fb2b50d2b06c18ba92c000ab7cf8e50aa9b7704fa418f03444e9c823d934 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-02T15:30:01.023 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.932Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-04-02T15:30:01.023 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.932Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-04-02T15:30:01.023 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.933Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-04-02T15:30:01.023 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.933Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-02T15:30:01.024 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.933Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-04-02T15:30:01.024 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.933Z caller=node_exporter.go:115 level=info collector=arp 2024-04-02T15:30:01.024 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.933Z caller=node_exporter.go:115 level=info collector=bcache 2024-04-02T15:30:01.024 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.933Z caller=node_exporter.go:115 level=info collector=bonding 2024-04-02T15:30:01.024 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.933Z caller=node_exporter.go:115 level=info collector=btrfs 2024-04-02T15:30:01.025 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:115 level=info collector=conntrack 2024-04-02T15:30:01.025 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:115 level=info collector=cpu 2024-04-02T15:30:01.025 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-04-02T15:30:01.025 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:115 level=info collector=diskstats 2024-04-02T15:30:01.025 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:115 level=info collector=dmi 2024-04-02T15:30:01.026 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:115 level=info collector=edac 2024-04-02T15:30:01.026 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:115 level=info collector=entropy 2024-04-02T15:30:01.026 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-04-02T15:30:01.026 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:115 level=info collector=filefd 2024-04-02T15:30:01.026 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:115 level=info collector=filesystem 2024-04-02T15:30:01.027 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:115 level=info collector=hwmon 2024-04-02T15:30:01.027 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:115 level=info collector=infiniband 2024-04-02T15:30:01.027 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:115 level=info collector=ipvs 2024-04-02T15:30:01.027 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:115 level=info collector=loadavg 2024-04-02T15:30:01.027 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:115 level=info collector=mdadm 2024-04-02T15:30:01.027 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:115 level=info collector=meminfo 2024-04-02T15:30:01.027 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:115 level=info collector=netclass 2024-04-02T15:30:01.028 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:115 level=info collector=netdev 2024-04-02T15:30:01.028 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:115 level=info collector=netstat 2024-04-02T15:30:01.028 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:115 level=info collector=nfs 2024-04-02T15:30:01.028 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:115 level=info collector=nfsd 2024-04-02T15:30:01.028 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:115 level=info collector=nvme 2024-04-02T15:30:01.028 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:115 level=info collector=os 2024-04-02T15:30:01.029 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-04-02T15:30:01.029 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:115 level=info collector=pressure 2024-04-02T15:30:01.029 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:115 level=info collector=rapl 2024-04-02T15:30:01.029 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:115 level=info collector=schedstat 2024-04-02T15:30:01.029 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:115 level=info collector=sockstat 2024-04-02T15:30:01.029 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:115 level=info collector=softnet 2024-04-02T15:30:01.029 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:115 level=info collector=stat 2024-04-02T15:30:01.029 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:115 level=info collector=tapestats 2024-04-02T15:30:01.030 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:115 level=info collector=textfile 2024-04-02T15:30:01.030 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-04-02T15:30:01.030 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:115 level=info collector=time 2024-04-02T15:30:01.030 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-04-02T15:30:01.030 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:115 level=info collector=uname 2024-04-02T15:30:01.030 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:115 level=info collector=vmstat 2024-04-02T15:30:01.030 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:115 level=info collector=xfs 2024-04-02T15:30:01.030 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:115 level=info collector=zfs 2024-04-02T15:30:01.030 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-04-02T15:30:01.031 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:00 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b[42865]: ts=2024-04-02T15:30:00.934Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-04-02T15:30:01.031 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:01 smithi106 bash[42813]: fb10fb2b50d2b06c18ba92c000ab7cf8e50aa9b7704fa418f03444e9c823d934 2024-04-02T15:30:01.247 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-02T15:30:01.247 INFO:teuthology.orchestra.run.smithi087.stdout:{"epoch":59,"fsid":"0400842c-f105-11ee-b647-cb9ed24678a4","created":"2024-04-02T15:24:24.701153+0000","modified":"2024-04-02T15:29:34.935122+0000","last_up_change":"2024-04-02T15:29:17.597118+0000","last_in_change":"2024-04-02T15:28:59.589376+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-02T15:27:25.144519+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"13d89f7c-4e33-4521-a94a-ff0658e7dccf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6802","nonce":2380439245},{"type":"v1","addr":"172.21.15.87:6803","nonce":2380439245}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6804","nonce":2380439245},{"type":"v1","addr":"172.21.15.87:6805","nonce":2380439245}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6808","nonce":2380439245},{"type":"v1","addr":"172.21.15.87:6809","nonce":2380439245}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6806","nonce":2380439245},{"type":"v1","addr":"172.21.15.87:6807","nonce":2380439245}]},"public_addr":"172.21.15.87:6803/2380439245","cluster_addr":"172.21.15.87:6805/2380439245","heartbeat_back_addr":"172.21.15.87:6809/2380439245","heartbeat_front_addr":"172.21.15.87:6807/2380439245","state":["exists","up"]},{"osd":1,"uuid":"f8e09079-dd20-411b-a160-5d6906d1d7ed","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6810","nonce":1909345406},{"type":"v1","addr":"172.21.15.87:6811","nonce":1909345406}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6812","nonce":1909345406},{"type":"v1","addr":"172.21.15.87:6813","nonce":1909345406}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6816","nonce":1909345406},{"type":"v1","addr":"172.21.15.87:6817","nonce":1909345406}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6814","nonce":1909345406},{"type":"v1","addr":"172.21.15.87:6815","nonce":1909345406}]},"public_addr":"172.21.15.87:6811/1909345406","cluster_addr":"172.21.15.87:6813/1909345406","heartbeat_back_addr":"172.21.15.87:6817/1909345406","heartbeat_front_addr":"172.21.15.87:6815/1909345406","state":["exists","up"]},{"osd":2,"uuid":"55868ded-a2a8-48c7-8fb2-d452831755d6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6818","nonce":2886422817},{"type":"v1","addr":"172.21.15.87:6819","nonce":2886422817}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6820","nonce":2886422817},{"type":"v1","addr":"172.21.15.87:6821","nonce":2886422817}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6824","nonce":2886422817},{"type":"v1","addr":"172.21.15.87:6825","nonce":2886422817}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6822","nonce":2886422817},{"type":"v1","addr":"172.21.15.87:6823","nonce":2886422817}]},"public_addr":"172.21.15.87:6819/2886422817","cluster_addr":"172.21.15.87:6821/2886422817","heartbeat_back_addr":"172.21.15.87:6825/2886422817","heartbeat_front_addr":"172.21.15.87:6823/2886422817","state":["exists","up"]},{"osd":3,"uuid":"fba679e6-6a92-4943-bbc9-3c446311dfc5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6826","nonce":1277310151},{"type":"v1","addr":"172.21.15.87:6827","nonce":1277310151}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6828","nonce":1277310151},{"type":"v1","addr":"172.21.15.87:6829","nonce":1277310151}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6832","nonce":1277310151},{"type":"v1","addr":"172.21.15.87:6833","nonce":1277310151}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.87:6830","nonce":1277310151},{"type":"v1","addr":"172.21.15.87:6831","nonce":1277310151}]},"public_addr":"172.21.15.87:6827/1277310151","cluster_addr":"172.21.15.87:6829/1277310151","heartbeat_back_addr":"172.21.15.87:6833/1277310151","heartbeat_front_addr":"172.21.15.87:6831/1277310151","state":["exists","up"]},{"osd":4,"uuid":"dcd6edd9-74b5-4ba9-9282-e9325c914835","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6800","nonce":2668368842},{"type":"v1","addr":"172.21.15.106:6801","nonce":2668368842}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6802","nonce":2668368842},{"type":"v1","addr":"172.21.15.106:6803","nonce":2668368842}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6806","nonce":2668368842},{"type":"v1","addr":"172.21.15.106:6807","nonce":2668368842}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6804","nonce":2668368842},{"type":"v1","addr":"172.21.15.106:6805","nonce":2668368842}]},"public_addr":"172.21.15.106:6801/2668368842","cluster_addr":"172.21.15.106:6803/2668368842","heartbeat_back_addr":"172.21.15.106:6807/2668368842","heartbeat_front_addr":"172.21.15.106:6805/2668368842","state":["exists","up"]},{"osd":5,"uuid":"4c205155-70f2-4d96-aebb-3d0e96ba5a95","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6808","nonce":2879909052},{"type":"v1","addr":"172.21.15.106:6809","nonce":2879909052}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6810","nonce":2879909052},{"type":"v1","addr":"172.21.15.106:6811","nonce":2879909052}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6814","nonce":2879909052},{"type":"v1","addr":"172.21.15.106:6815","nonce":2879909052}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6812","nonce":2879909052},{"type":"v1","addr":"172.21.15.106:6813","nonce":2879909052}]},"public_addr":"172.21.15.106:6809/2879909052","cluster_addr":"172.21.15.106:6811/2879909052","heartbeat_back_addr":"172.21.15.106:6815/2879909052","heartbeat_front_addr":"172.21.15.106:6813/2879909052","state":["exists","up"]},{"osd":6,"uuid":"391fb1af-ab61-4a21-8bf3-da6f0117756e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6816","nonce":125323095},{"type":"v1","addr":"172.21.15.106:6817","nonce":125323095}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6818","nonce":125323095},{"type":"v1","addr":"172.21.15.106:6819","nonce":125323095}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6822","nonce":125323095},{"type":"v1","addr":"172.21.15.106:6823","nonce":125323095}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6820","nonce":125323095},{"type":"v1","addr":"172.21.15.106:6821","nonce":125323095}]},"public_addr":"172.21.15.106:6817/125323095","cluster_addr":"172.21.15.106:6819/125323095","heartbeat_back_addr":"172.21.15.106:6823/125323095","heartbeat_front_addr":"172.21.15.106:6821/125323095","state":["exists","up"]},{"osd":7,"uuid":"780bffc8-5404-40eb-860f-26c6e0c773b6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6824","nonce":256892273},{"type":"v1","addr":"172.21.15.106:6825","nonce":256892273}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6826","nonce":256892273},{"type":"v1","addr":"172.21.15.106:6827","nonce":256892273}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6830","nonce":256892273},{"type":"v1","addr":"172.21.15.106:6831","nonce":256892273}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.106:6828","nonce":256892273},{"type":"v1","addr":"172.21.15.106:6829","nonce":256892273}]},"public_addr":"172.21.15.106:6825/256892273","cluster_addr":"172.21.15.106:6827/256892273","heartbeat_back_addr":"172.21.15.106:6831/256892273","heartbeat_front_addr":"172.21.15.106:6829/256892273","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T15:26:37.310106+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T15:26:59.114255+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T15:27:22.084772+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T15:27:45.261604+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T15:28:08.046780+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T15:28:31.088909+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T15:28:53.265880+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T15:29:15.668954+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.87:0/3267436387":"2024-04-03T15:29:34.935088+0000","172.21.15.87:6800/2108248912":"2024-04-03T15:29:34.935088+0000","172.21.15.87:0/2189995990":"2024-04-03T15:25:16.785348+0000","172.21.15.87:0/1859893958":"2024-04-03T15:25:16.785348+0000","172.21.15.87:6800/3965460628":"2024-04-03T15:25:16.785348+0000","172.21.15.87:0/4051190714":"2024-04-03T15:29:34.935088+0000","172.21.15.87:6800/2380456996":"2024-04-03T15:24:54.083807+0000","172.21.15.87:0/2150387812":"2024-04-03T15:29:34.935088+0000","172.21.15.87:0/2076084295":"2024-04-03T15:25:16.785348+0000","172.21.15.87:6801/2380456996":"2024-04-03T15:24:54.083807+0000","172.21.15.87:6801/3965460628":"2024-04-03T15:25:16.785348+0000","172.21.15.87:0/3045002547":"2024-04-03T15:24:54.083807+0000","172.21.15.87:0/2722718326":"2024-04-03T15:24:54.083807+0000","172.21.15.87:6801/2108248912":"2024-04-03T15:29:34.935088+0000","172.21.15.87:0/3328279098":"2024-04-03T15:29:34.935088+0000","172.21.15.87:0/593333980":"2024-04-03T15:24:54.083807+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-02T15:30:01.366 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:30:01 smithi106 systemd[1]: Started Ceph node-exporter.b for 0400842c-f105-11ee-b647-cb9ed24678a4. 2024-04-02T15:30:01.874 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph tell osd.0 flush_pg_stats 2024-04-02T15:30:01.874 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph tell osd.1 flush_pg_stats 2024-04-02T15:30:01.875 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph tell osd.2 flush_pg_stats 2024-04-02T15:30:01.875 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph tell osd.3 flush_pg_stats 2024-04-02T15:30:01.875 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph tell osd.4 flush_pg_stats 2024-04-02T15:30:01.876 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph tell osd.5 flush_pg_stats 2024-04-02T15:30:01.876 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph tell osd.6 flush_pg_stats 2024-04-02T15:30:01.877 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph tell osd.7 flush_pg_stats 2024-04-02T15:30:02.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:02 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:02.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:02 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:30:02.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:02 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:30:02.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:02 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:30:02.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:02 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:02.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:02 smithi087 ceph-mon[24900]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-02T15:30:02.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:02 smithi087 ceph-mon[24900]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-02T15:30:02.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:02 smithi087 ceph-mon[24900]: Reconfiguring daemon prometheus.a on smithi106 2024-04-02T15:30:02.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:02 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/248217587' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T15:30:02.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:02 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:02.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:02 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:30:02.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:02 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:30:02.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:02 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:30:02.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:02 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:02.122 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:02 smithi087 ceph-mon[29707]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-02T15:30:02.122 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:02 smithi087 ceph-mon[29707]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-02T15:30:02.122 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:02 smithi087 ceph-mon[29707]: Reconfiguring daemon prometheus.a on smithi106 2024-04-02T15:30:02.122 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:02 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/248217587' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T15:30:02.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:02 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:02.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:02 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:30:02.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:02 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:30:02.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:02 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:30:02.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:02 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:02.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:02 smithi106 ceph-mon[25843]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-02T15:30:02.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:02 smithi106 ceph-mon[25843]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-02T15:30:02.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:02 smithi106 ceph-mon[25843]: Reconfiguring daemon prometheus.a on smithi106 2024-04-02T15:30:02.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:02 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/248217587' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T15:30:02.569 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:30:02.649 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:30:02.649 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:30:02.652 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:30:02.656 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:30:02.656 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:30:02.657 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:30:03.125 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:03 smithi087 ceph-mon[29707]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:03.125 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:03 smithi087 ceph-mon[24900]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:03.352 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:03 smithi106 ceph-mon[25843]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:03.989 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:30:03 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:30:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:30:05.251 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:05 smithi106 ceph-mon[25843]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:05.252 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:05 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:05.252 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:05 smithi106 bash[43150]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus.a" found: no such container 2024-04-02T15:30:05.309 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:05 smithi087 ceph-mon[24900]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:05.310 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:05 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:05.310 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:30:05 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:30:05] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:30:05.313 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:05 smithi087 ceph-mon[29707]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:05.313 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:05 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:05.502 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:05 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[41992]: ts=2024-04-02T15:30:05.255Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-02T15:30:05.502 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:05 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[41992]: ts=2024-04-02T15:30:05.255Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-02T15:30:05.502 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:05 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[41992]: ts=2024-04-02T15:30:05.255Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-02T15:30:05.503 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:05 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[41992]: ts=2024-04-02T15:30:05.255Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-02T15:30:05.503 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:05 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[41992]: ts=2024-04-02T15:30:05.255Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-02T15:30:05.503 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:05 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[41992]: ts=2024-04-02T15:30:05.255Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-02T15:30:05.503 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:05 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[41992]: ts=2024-04-02T15:30:05.255Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-02T15:30:05.503 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:05 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[41992]: ts=2024-04-02T15:30:05.255Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-02T15:30:05.503 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:05 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[41992]: ts=2024-04-02T15:30:05.255Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-02T15:30:05.503 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:05 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[41992]: ts=2024-04-02T15:30:05.256Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-02T15:30:05.503 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:05 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[41992]: ts=2024-04-02T15:30:05.256Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-02T15:30:05.503 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:05 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[41992]: ts=2024-04-02T15:30:05.256Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-02T15:30:05.503 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:05 smithi106 podman[43161]: 2024-04-02 15:30:05.265861499 +0000 UTC m=+0.137396185 container died 9cda5cca4c5d2b85b5b487c5bc7c7bd73d3f3e5fbdecd2166f8e6a8498482062 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T15:30:05.827 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:05 smithi106 podman[43161]: 2024-04-02 15:30:05.596533218 +0000 UTC m=+0.468067890 container cleanup 9cda5cca4c5d2b85b5b487c5bc7c7bd73d3f3e5fbdecd2166f8e6a8498482062 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T15:30:05.827 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:05 smithi106 bash[43161]: ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a 2024-04-02T15:30:05.827 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:05 smithi106 bash[43198]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus.a" found: no such container 2024-04-02T15:30:06.116 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:05 smithi106 podman[43175]: 2024-04-02 15:30:05.831171299 +0000 UTC m=+0.567056222 container remove 9cda5cca4c5d2b85b5b487c5bc7c7bd73d3f3e5fbdecd2166f8e6a8498482062 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T15:30:06.116 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:05 smithi106 systemd[1]: ceph-0400842c-f105-11ee-b647-cb9ed24678a4@prometheus.a.service: Deactivated successfully. 2024-04-02T15:30:06.116 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:05 smithi106 systemd[1]: Stopped Ceph prometheus.a for 0400842c-f105-11ee-b647-cb9ed24678a4. 2024-04-02T15:30:06.116 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:05 smithi106 systemd[1]: Starting Ceph prometheus.a for 0400842c-f105-11ee-b647-cb9ed24678a4... 2024-04-02T15:30:06.464 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:06 smithi106 ceph-mon[25843]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:06.465 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:06 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:06.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:06 smithi087 ceph-mon[29707]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:06.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:06 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:06.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:06 smithi087 ceph-mon[24900]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:06.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:06 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:06.757 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:06 smithi106 podman[43356]: 2024-04-02 15:30:06.601696551 +0000 UTC m=+0.022883402 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-02T15:30:07.023 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:06 smithi106 podman[43356]: 2024-04-02 15:30:06.761199108 +0000 UTC m=+0.182385918 container create 413397b51ac34273e87ef1c91905012d78db7c80a6e171ae552a8e219e59f967 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T15:30:07.023 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:06 smithi106 podman[43356]: 2024-04-02 15:30:06.98016855 +0000 UTC m=+0.401355369 container init 413397b51ac34273e87ef1c91905012d78db7c80a6e171ae552a8e219e59f967 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T15:30:07.023 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:06 smithi106 podman[43356]: 2024-04-02 15:30:06.982914688 +0000 UTC m=+0.404101500 container start 413397b51ac34273e87ef1c91905012d78db7c80a6e171ae552a8e219e59f967 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T15:30:07.023 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:07 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[43401]: ts=2024-04-02T15:30:07.005Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-02T15:30:07.023 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:07 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[43401]: ts=2024-04-02T15:30:07.005Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-02T15:30:07.023 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:07 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[43401]: ts=2024-04-02T15:30:07.005Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-02T15:30:07.023 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:07 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[43401]: ts=2024-04-02T15:30:07.005Z caller=main.go:518 level=info host_details="(Linux 5.14.0-434.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Mar 26 19:20:47 UTC 2024 x86_64 smithi106 (none))" 2024-04-02T15:30:07.023 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:07 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[43401]: ts=2024-04-02T15:30:07.005Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-02T15:30:07.023 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:07 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[43401]: ts=2024-04-02T15:30:07.005Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-02T15:30:07.023 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:07 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[43401]: ts=2024-04-02T15:30:07.007Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-02T15:30:07.023 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:07 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[43401]: ts=2024-04-02T15:30:07.007Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-02T15:30:07.023 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:07 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[43401]: ts=2024-04-02T15:30:07.008Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-02T15:30:07.024 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:07 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[43401]: ts=2024-04-02T15:30:07.009Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-02T15:30:07.024 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:07 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[43401]: ts=2024-04-02T15:30:07.009Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.414µs 2024-04-02T15:30:07.024 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:07 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[43401]: ts=2024-04-02T15:30:07.009Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-02T15:30:07.024 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:07 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[43401]: ts=2024-04-02T15:30:07.010Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-04-02T15:30:07.024 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:07 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[43401]: ts=2024-04-02T15:30:07.010Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-04-02T15:30:07.024 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:07 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[43401]: ts=2024-04-02T15:30:07.010Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=21.883µs wal_replay_duration=232.2µs total_replay_duration=266.912µs 2024-04-02T15:30:07.024 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:07 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[43401]: ts=2024-04-02T15:30:07.010Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-02T15:30:07.024 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:07 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[43401]: ts=2024-04-02T15:30:07.010Z caller=main.go:947 level=info msg="TSDB started" 2024-04-02T15:30:07.024 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:07 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[43401]: ts=2024-04-02T15:30:07.010Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-02T15:30:07.024 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:07 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[43401]: ts=2024-04-02T15:30:07.026Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.515619ms db_storage=603ns remote_storage=1.227µs web_handler=195ns query_engine=676ns scrape=359.795µs scrape_sd=21.461µs notify=912ns notify_sd=2.063µs rules=14.942864ms 2024-04-02T15:30:07.024 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:07 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[43401]: ts=2024-04-02T15:30:07.026Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-02T15:30:07.366 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:07 smithi106 bash[43356]: 413397b51ac34273e87ef1c91905012d78db7c80a6e171ae552a8e219e59f967 2024-04-02T15:30:07.366 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:30:07 smithi106 systemd[1]: Started Ceph prometheus.a for 0400842c-f105-11ee-b647-cb9ed24678a4. 2024-04-02T15:30:08.309 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:08 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:08.310 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:08 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T15:30:08.310 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:08 smithi106 ceph-mon[25843]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T15:30:08.310 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:08 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.106:9095"}]: dispatch 2024-04-02T15:30:08.310 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:08 smithi106 ceph-mon[25843]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.106:9095"}]: dispatch 2024-04-02T15:30:08.310 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:08 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:08.310 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:08 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:30:08.310 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:08 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:30:08.310 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:08 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:30:08.310 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:08 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:08.310 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:08 smithi106 ceph-mon[25843]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-02T15:30:08.310 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:08 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:30:08.310 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:08 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:30:08.311 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:08 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:30:08.311 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:08 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:08.311 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:08 smithi106 ceph-mon[25843]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-02T15:30:08.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:08 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:08.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:08 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T15:30:08.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:08 smithi087 ceph-mon[24900]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T15:30:08.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:08 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.106:9095"}]: dispatch 2024-04-02T15:30:08.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:08 smithi087 ceph-mon[24900]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.106:9095"}]: dispatch 2024-04-02T15:30:08.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:08 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:08.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:08 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:30:08.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:08 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:30:08.373 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:08 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:30:08.373 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:08 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:08.373 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:08 smithi087 ceph-mon[24900]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-02T15:30:08.373 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:08 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:30:08.373 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:08 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:30:08.373 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:08 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:30:08.373 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:08 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:08.373 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:08 smithi087 ceph-mon[24900]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-02T15:30:08.373 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:08 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:08.373 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:08 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T15:30:08.373 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:08 smithi087 ceph-mon[29707]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T15:30:08.373 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:08 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.106:9095"}]: dispatch 2024-04-02T15:30:08.374 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:08 smithi087 ceph-mon[29707]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.106:9095"}]: dispatch 2024-04-02T15:30:08.374 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:08 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:08.374 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:08 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:30:08.374 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:08 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:30:08.374 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:08 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:30:08.374 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:08 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:08.374 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:08 smithi087 ceph-mon[29707]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-02T15:30:08.374 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:08 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:30:08.374 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:08 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:30:08.374 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:08 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:30:08.374 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:08 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:08.374 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:08 smithi087 ceph-mon[29707]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-02T15:30:08.693 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:30:09.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:09 smithi106 ceph-mon[25843]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:09.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:09 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:09.370 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:30:09 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:30:09] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:30:09.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:09 smithi087 ceph-mon[29707]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:09.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:09 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:09.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:09 smithi087 ceph-mon[24900]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:09.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:09 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:09.870 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:30:09 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:30:09] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:30:10.795 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:10 smithi087 ceph-mon[29707]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:10.795 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:10 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:10.795 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:10 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:10.795 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:10 smithi087 ceph-mon[24900]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:10.795 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:10 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:10.795 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:10 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:11.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:10 smithi106 ceph-mon[25843]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:11.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:10 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:11.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:10 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:12.541 INFO:teuthology.orchestra.run.smithi087.stdout:133143986201 2024-04-02T15:30:12.541 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.4 2024-04-02T15:30:12.934 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:12 smithi087 ceph-mon[29707]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:12.935 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:12 smithi087 ceph-mon[24900]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:13.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:12 smithi106 ceph-mon[25843]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:13.974 INFO:teuthology.orchestra.run.smithi087.stdout:55834574888 2024-04-02T15:30:13.974 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.1 2024-04-02T15:30:14.116 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:30:13 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:30:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:30:15.035 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:14 smithi087 ceph-mon[29707]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:15.035 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:14 smithi087 ceph-mon[24900]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:15.057 INFO:teuthology.orchestra.run.smithi087.stdout:167503724567 2024-04-02T15:30:15.058 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.5 2024-04-02T15:30:15.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:14 smithi106 ceph-mon[25843]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:16.454 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:30:16 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:30:16] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:30:16.454 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:30:16 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:30:16] "GET /metrics HTTP/1.1" 200 191489 "" "Prometheus/2.33.4" 2024-04-02T15:30:16.789 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:16 smithi087 ceph-mon[29707]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:16.789 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:16 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:16.790 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:16 smithi087 ceph-mon[24900]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:16.790 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:16 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:17.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:16 smithi106 ceph-mon[25843]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:17.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:16 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:18.327 INFO:teuthology.orchestra.run.smithi087.stdout:77309411364 2024-04-02T15:30:18.327 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.2 2024-04-02T15:30:18.951 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:18 smithi087 ceph-mon[24900]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:18.951 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:18 smithi087 ceph-mon[29707]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:19.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:18 smithi106 ceph-mon[25843]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:19.939 INFO:teuthology.orchestra.run.smithi087.stdout:231928233999 2024-04-02T15:30:19.939 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.7 2024-04-02T15:30:20.224 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:30:20.225 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:30:20.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:20 smithi106 ceph-mon[25843]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:20.619 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:20 smithi087 ceph-mon[29707]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:20.619 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:20 smithi087 ceph-mon[24900]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:20.642 INFO:teuthology.orchestra.run.smithi087.stdout:197568495633 2024-04-02T15:30:20.643 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.6 2024-04-02T15:30:21.102 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:30:21.102 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:30:21.630 INFO:teuthology.orchestra.run.smithi087.stdout:111669149728 2024-04-02T15:30:21.631 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.3 2024-04-02T15:30:22.518 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:22 smithi087 ceph-mon[29707]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:22.518 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:22 smithi087 ceph-mon[24900]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:22.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:22 smithi106 ceph-mon[25843]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:24.069 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:30:23 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:30:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:30:24.139 INFO:teuthology.orchestra.run.smithi087.stdout:167503724569 2024-04-02T15:30:24.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:24 smithi106 ceph-mon[25843]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:24.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:24 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/413600071' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-02T15:30:24.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:24 smithi087 ceph-mon[29707]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:24.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:24 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/413600071' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-02T15:30:24.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:24 smithi087 ceph-mon[24900]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:24.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:24 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/413600071' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-02T15:30:24.517 INFO:teuthology.orchestra.run.smithi087.stdout:34359738414 2024-04-02T15:30:24.518 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.0 2024-04-02T15:30:26.071 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:30:25 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:30:25] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:30:26.095 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724567 got 167503724569 for osd.5 2024-04-02T15:30:26.095 DEBUG:teuthology.parallel:result is None 2024-04-02T15:30:26.220 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:30:26.220 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:30:26.221 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:30:26.616 INFO:teuthology.orchestra.run.smithi087.stdout:197568495635 2024-04-02T15:30:26.617 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:30:26 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:30:26] "GET /metrics HTTP/1.1" 200 191476 "" "Prometheus/2.33.4" 2024-04-02T15:30:26.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:26 smithi087 ceph-mon[24900]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:26.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:26 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:26.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:26 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:26.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:26 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/3304706451' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-02T15:30:26.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:26 smithi087 ceph-mon[29707]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:26.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:26 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:26.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:26 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:26.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:26 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/3304706451' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-02T15:30:27.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:26 smithi106 ceph-mon[25843]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:27.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:26 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:27.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:26 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:27.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:26 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/3304706451' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-02T15:30:29.111 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:28 smithi087 ceph-mon[29707]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:29.111 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:28 smithi087 ceph-mon[24900]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:29.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:28 smithi106 ceph-mon[25843]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:29.557 INFO:teuthology.orchestra.run.smithi087.stdout:133143986205 2024-04-02T15:30:30.470 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:30 smithi087 ceph-mon[29707]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:30.470 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:30 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/2193635656' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-02T15:30:30.470 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:30 smithi087 ceph-mon[24900]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:30.470 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:30 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/2193635656' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-02T15:30:30.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:30 smithi106 ceph-mon[25843]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:30.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:30 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/2193635656' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-02T15:30:31.336 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495633 got 197568495635 for osd.6 2024-04-02T15:30:31.336 DEBUG:teuthology.parallel:result is None 2024-04-02T15:30:31.738 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986201 got 133143986205 for osd.4 2024-04-02T15:30:31.738 DEBUG:teuthology.parallel:result is None 2024-04-02T15:30:32.135 INFO:teuthology.orchestra.run.smithi087.stdout:55834574892 2024-04-02T15:30:32.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:32 smithi087 ceph-mon[29707]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:32.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:32 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/2840380102' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-02T15:30:32.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:32 smithi087 ceph-mon[24900]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:32.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:32 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/2840380102' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-02T15:30:32.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:32 smithi106 ceph-mon[25843]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:32.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:32 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/2840380102' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-02T15:30:33.647 INFO:teuthology.orchestra.run.smithi087.stdout:231928234002 2024-04-02T15:30:34.116 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:30:33 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:30:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:30:34.346 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928233999 got 231928234002 for osd.7 2024-04-02T15:30:34.347 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:30:34.347 DEBUG:teuthology.parallel:result is None 2024-04-02T15:30:34.572 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:34 smithi087 ceph-mon[24900]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:34.572 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:34 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/279192505' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-02T15:30:34.572 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:34 smithi087 ceph-mon[29707]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:34.573 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:34 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/279192505' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-02T15:30:34.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:34 smithi106 ceph-mon[25843]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:34.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:34 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/279192505' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-02T15:30:35.722 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574888 got 55834574892 for osd.1 2024-04-02T15:30:35.723 DEBUG:teuthology.parallel:result is None 2024-04-02T15:30:36.417 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:36 smithi087 ceph-mon[24900]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:36.417 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:30:36.417 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:30:36.418 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:30:36 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:30:36] "GET /metrics HTTP/1.1" 200 191475 "" "Prometheus/2.33.4" 2024-04-02T15:30:36.418 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:30:36 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:30:36] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:30:36.418 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:36 smithi087 ceph-mon[29707]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:36.418 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:30:36.418 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:30:36.446 INFO:teuthology.orchestra.run.smithi087.stdout:77309411367 2024-04-02T15:30:36.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:36 smithi106 ceph-mon[25843]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:36.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:30:36.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:30:36.900 INFO:teuthology.orchestra.run.smithi087.stdout:34359738417 2024-04-02T15:30:37.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:37 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:37.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:37 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/4256353967' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-02T15:30:37.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:37 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/3687990841' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-02T15:30:37.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:37 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:37.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:37 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/4256353967' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-02T15:30:37.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:37 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/3687990841' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-02T15:30:37.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:37 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:37.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:37 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/4256353967' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-02T15:30:37.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:37 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/3687990841' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-02T15:30:37.763 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738414 got 34359738417 for osd.0 2024-04-02T15:30:37.764 DEBUG:teuthology.parallel:result is None 2024-04-02T15:30:38.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:38 smithi106 ceph-mon[25843]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:38.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:38 smithi087 ceph-mon[29707]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:38.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:38 smithi087 ceph-mon[24900]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:38.670 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411364 got 77309411367 for osd.2 2024-04-02T15:30:38.670 DEBUG:teuthology.parallel:result is None 2024-04-02T15:30:39.796 INFO:teuthology.orchestra.run.smithi087.stdout:111669149732 2024-04-02T15:30:40.487 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:40 smithi106 ceph-mon[25843]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:40.487 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:40 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/1332328791' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-02T15:30:40.498 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:40 smithi087 ceph-mon[29707]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:40.499 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:40 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/1332328791' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-02T15:30:40.499 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:40 smithi087 ceph-mon[24900]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:40.499 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:40 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/1332328791' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-02T15:30:40.733 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149728 got 111669149732 for osd.3 2024-04-02T15:30:40.733 DEBUG:teuthology.parallel:result is None 2024-04-02T15:30:40.734 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-02T15:30:40.734 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph pg dump --format=json 2024-04-02T15:30:41.403 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:30:42.370 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:30:42 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:30:42] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:30:43.350 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:43 smithi087 ceph-mon[29707]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:43.350 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:43 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:43.351 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:43 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:43.351 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:43 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:30:43.351 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:43 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:30:43.351 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:43 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:30:43.351 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:43 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:43.351 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:43 smithi087 ceph-mon[29707]: Deploying daemon alertmanager.a on smithi087 2024-04-02T15:30:43.351 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:43 smithi087 ceph-mon[24900]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:43.351 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:43 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:43.351 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:43 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:43.351 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:43 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:30:43.351 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:43 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:30:43.351 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:43 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:30:43.352 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:43 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:43.352 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:43 smithi087 ceph-mon[24900]: Deploying daemon alertmanager.a on smithi087 2024-04-02T15:30:43.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:43 smithi106 ceph-mon[25843]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:43.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:43 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:43.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:43 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:43.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:43 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:30:43.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:43 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:30:43.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:43 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:30:43.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:43 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:43.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:43 smithi106 ceph-mon[25843]: Deploying daemon alertmanager.a on smithi087 2024-04-02T15:30:43.643 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-02T15:30:43.644 INFO:teuthology.orchestra.run.smithi087.stderr:dumped all 2024-04-02T15:30:44.116 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:30:43 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:30:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:30:44.286 INFO:teuthology.orchestra.run.smithi087.stdout:{"pg_ready":true,"pg_map":{"version":37,"stamp":"2024-04-02T15:30:43.226099+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48808,"kb_used_data":6056,"kb_used_omap":0,"kb_used_meta":42688,"kb_avail":749912408,"statfs":{"total":767960285184,"available":767910305792,"internally_reserved":0,"allocated":6201344,"data_stored":3473654,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43712512},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001848"},"pg_stats":[{"pgid":"1.0","version":"59'128","reported_seq":267,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-04-02T15:29:35.303161+0000","last_change":"2024-04-02T15:29:21.613260+0000","last_active":"2024-04-02T15:29:35.303161+0000","last_peered":"2024-04-02T15:29:35.303161+0000","last_clean":"2024-04-02T15:29:35.303161+0000","last_became_active":"2024-04-02T15:29:21.612932+0000","last_became_peered":"2024-04-02T15:29:21.612932+0000","last_unstale":"2024-04-02T15:29:35.303161+0000","last_undegraded":"2024-04-02T15:29:35.303161+0000","last_fullsized":"2024-04-02T15:29:35.303161+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-02T15:27:25.608662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-02T15:27:25.608662+0000","last_clean_scrub_stamp":"2024-04-02T15:27:25.608662+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-03T18:32:49.541893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234004,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6084,"kb_used_data":1084,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739068,"statfs":{"total":95995035648,"available":95988805632,"internally_reserved":0,"allocated":1110016,"data_stored":764977,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 15:30:20 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.41599999999999998}]},{"osd":1,"last update":"Tue Apr 2 15:30:20 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.51200000000000001}]},{"osd":2,"last update":"Tue Apr 2 15:30:20 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.54000000000000004}]},{"osd":3,"last update":"Tue Apr 2 15:30:20 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.47499999999999998}]},{"osd":4,"last update":"Tue Apr 2 15:30:20 2024","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.45000000000000001}]},{"osd":5,"last update":"Tue Apr 2 15:30:20 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.438,"15min":0.438},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.57299999999999995}]},{"osd":6,"last update":"Tue Apr 2 15:30:20 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.58799999999999997}]}]},{"osd":6,"up_from":46,"seq":197568495639,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6140,"kb_used_data":1076,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739012,"statfs":{"total":95995035648,"available":95988748288,"internally_reserved":0,"allocated":1101824,"data_stored":764389,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 15:29:55 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.437}]},{"osd":1,"last update":"Tue Apr 2 15:29:55 2024","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.504},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.46600000000000003}]},{"osd":2,"last update":"Tue Apr 2 15:29:55 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.91600000000000004,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.498}]},{"osd":3,"last update":"Tue Apr 2 15:29:55 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.49099999999999999}]},{"osd":4,"last update":"Tue Apr 2 15:29:55 2024","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.48499999999999999}]},{"osd":5,"last update":"Tue Apr 2 15:29:55 2024","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.442,"5min":0.442,"15min":0.442},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.45500000000000002}]},{"osd":7,"last update":"Tue Apr 2 15:30:18 2024","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.56399999999999995}]}]},{"osd":1,"up_from":13,"seq":55834574894,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6332,"kb_used_data":564,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738820,"statfs":{"total":95995035648,"available":95988551680,"internally_reserved":0,"allocated":577536,"data_stored":236049,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 15:30:10 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.35899999999999999,"15min":0.34100000000000003},"min":{"1min":0.221,"5min":0.22,"15min":0.22},"max":{"1min":0.624,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.28000000000000003},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.374,"15min":0.36199999999999999},"min":{"1min":0.247,"5min":0.224,"15min":0.224},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.30099999999999999}]},{"osd":2,"last update":"Tue Apr 2 15:30:26 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.41999999999999998,"15min":0.41799999999999998},"min":{"1min":0.28299999999999997,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.68400000000000005,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.374},{"interface":"front","average":{"1min":0.38200000000000001,"5min":0.39400000000000002,"15min":0.39200000000000002},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.29099999999999998}]},{"osd":3,"last update":"Tue Apr 2 15:29:50 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.441,"15min":0.44700000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.317},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.48299999999999998,"15min":0.48499999999999999},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.40000000000000002}]},{"osd":4,"last update":"Tue Apr 2 15:30:10 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.55000000000000004,"15min":0.55400000000000005},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.71299999999999997,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56899999999999995,"15min":0.57099999999999995},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.53000000000000003}]},{"osd":5,"last update":"Tue Apr 2 15:30:35 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.58299999999999996,"15min":0.57899999999999996},"min":{"1min":0.39600000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.73099999999999998,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.48199999999999998}]},{"osd":6,"last update":"Tue Apr 2 15:29:55 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.51600000000000001}]},{"osd":7,"last update":"Tue Apr 2 15:30:20 2024","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.60499999999999998}]}]},{"osd":0,"up_from":8,"seq":34359738419,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6916,"kb_used_data":1084,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738236,"statfs":{"total":95995035648,"available":95987953664,"internally_reserved":0,"allocated":1110016,"data_stored":764977,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Tue Apr 2 15:30:04 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.35899999999999999,"15min":0.34999999999999998},"min":{"1min":0.23400000000000001,"5min":0.215,"15min":0.215},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.36699999999999999,"15min":0.35099999999999998},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":0.57699999999999996,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.36599999999999999}]},{"osd":2,"last update":"Tue Apr 2 15:30:32 2024","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.38400000000000001,"15min":0.379},"min":{"1min":0.27000000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.39900000000000002,"15min":0.39400000000000002},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.52400000000000002,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.42599999999999999}]},{"osd":3,"last update":"Tue Apr 2 15:29:47 2024","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.438},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.44600000000000001,"15min":0.44400000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.49099999999999999}]},{"osd":4,"last update":"Tue Apr 2 15:30:12 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.56499999999999995,"15min":0.56899999999999995},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.60099999999999998,"15min":0.60399999999999998},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.76100000000000001,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.58399999999999996}]},{"osd":5,"last update":"Tue Apr 2 15:30:36 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.38800000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58499999999999996,"15min":0.58599999999999997},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.71599999999999997,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.68300000000000005}]},{"osd":6,"last update":"Tue Apr 2 15:29:56 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.60499999999999998}]},{"osd":7,"last update":"Tue Apr 2 15:30:23 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.621},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.66700000000000004}]}]},{"osd":2,"up_from":18,"seq":77309411369,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6200,"kb_used_data":560,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738952,"statfs":{"total":95995035648,"available":95988686848,"internally_reserved":0,"allocated":573440,"data_stored":235438,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 15:30:33 2024","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.36699999999999999,"15min":0.35799999999999998},"min":{"1min":0.24299999999999999,"5min":0.224,"15min":0.224},"max":{"1min":0.47299999999999998,"5min":0.56799999999999995,"15min":0.56799999999999995},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.35099999999999998,"5min":0.36599999999999999,"15min":0.36199999999999999},"min":{"1min":0.25800000000000001,"5min":0.252,"15min":0.252},"max":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.40400000000000003}]},{"osd":1,"last update":"Tue Apr 2 15:30:33 2024","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.38300000000000001,"15min":0.379},"min":{"1min":0.27900000000000003,"5min":0.248,"15min":0.248},"max":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.35299999999999998,"5min":0.371,"15min":0.36699999999999999},"min":{"1min":0.24299999999999999,"5min":0.221,"15min":0.221},"max":{"1min":0.48599999999999999,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.47199999999999998}]},{"osd":3,"last update":"Tue Apr 2 15:29:50 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"min":{"1min":0.30399999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.53700000000000003,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.441,"5min":0.41999999999999998,"15min":0.41599999999999998},"min":{"1min":0.28000000000000003,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.623,"5min":0.623,"15min":0.623},"last":0.442}]},{"osd":4,"last update":"Tue Apr 2 15:30:13 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.91400000000000003,"15min":0.98199999999999998},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.67500000000000004,"5min":8.4489999999999998,"15min":8.4489999999999998},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.60299999999999998,"15min":0.61499999999999999},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.73399999999999999,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.71699999999999997}]},{"osd":5,"last update":"Tue Apr 2 15:29:36 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.61699999999999999}]},{"osd":6,"last update":"Tue Apr 2 15:29:55 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.60899999999999999}]},{"osd":7,"last update":"Tue Apr 2 15:30:19 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.64100000000000001}]}]},{"osd":3,"up_from":26,"seq":111669149733,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5692,"kb_used_data":564,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739460,"statfs":{"total":95995035648,"available":95989207040,"internally_reserved":0,"allocated":577536,"data_stored":236049,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 15:29:49 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.36199999999999999,"15min":0.35399999999999998},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.28199999999999997},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.38800000000000001,"15min":0.38300000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.29899999999999999}]},{"osd":1,"last update":"Tue Apr 2 15:29:49 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.41599999999999998,"15min":0.41199999999999998},"min":{"1min":0.27900000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.251},{"interface":"front","average":{"1min":0.435,"5min":0.41499999999999998,"15min":0.41099999999999998},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.318}]},{"osd":2,"last update":"Tue Apr 2 15:29:49 2024","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.42399999999999999,"15min":0.41599999999999998},"min":{"1min":0.315,"5min":0.308,"15min":0.308},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.434,"5min":0.42799999999999999,"15min":0.42699999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.28999999999999998}]},{"osd":4,"last update":"Tue Apr 2 15:30:15 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.58799999999999997,"15min":0.58399999999999996},"min":{"1min":0.40999999999999998,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.89700000000000002,"5min":1.0309999999999999,"15min":1.0309999999999999},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.56299999999999994,"15min":0.55500000000000005},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.54200000000000004}]},{"osd":5,"last update":"Tue Apr 2 15:30:36 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.60299999999999998,"15min":0.60399999999999998},"min":{"1min":0.41899999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.84099999999999997,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.502}]},{"osd":6,"last update":"Tue Apr 2 15:29:55 2024","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":1.0069999999999999,"5min":1.0069999999999999,"15min":1.0069999999999999},"last":0.71599999999999997}]},{"osd":7,"last update":"Tue Apr 2 15:30:20 2024","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.58799999999999997}]}]},{"osd":4,"up_from":31,"seq":133143986208,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5756,"kb_used_data":564,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739396,"statfs":{"total":95995035648,"available":95989141504,"internally_reserved":0,"allocated":577536,"data_stored":236049,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 15:30:15 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.53000000000000003,"15min":0.53200000000000003},"min":{"1min":0.29499999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.75,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.499,"5min":0.51300000000000001,"15min":0.51500000000000001},"min":{"1min":0.36699999999999999,"5min":0.307,"15min":0.307},"max":{"1min":0.68600000000000005,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.49399999999999999}]},{"osd":1,"last update":"Tue Apr 2 15:30:15 2024","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.504,"15min":0.51000000000000001},"min":{"1min":0.35299999999999998,"5min":0.309,"15min":0.309},"max":{"1min":0.623,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.53600000000000003,"15min":0.54000000000000004},"min":{"1min":0.33800000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.83899999999999997,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.51400000000000001}]},{"osd":2,"last update":"Tue Apr 2 15:30:15 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.54300000000000004,"15min":0.54100000000000004},"min":{"1min":0.40500000000000003,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.80000000000000004,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.499,"5min":0.53800000000000003,"15min":0.54400000000000004},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.73099999999999998,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.59699999999999998}]},{"osd":3,"last update":"Tue Apr 2 15:30:15 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.52900000000000003,"15min":0.53600000000000003},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.72599999999999998,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.54200000000000004,"15min":0.54700000000000004},"min":{"1min":0.34899999999999998,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.72699999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.46700000000000003}]},{"osd":5,"last update":"Tue Apr 2 15:30:36 2024","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.40799999999999997,"15min":0.40000000000000002},"min":{"1min":0.309,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.65000000000000002,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.441,"15min":0.433},"min":{"1min":0.33300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.79800000000000004,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.45100000000000001}]},{"osd":6,"last update":"Tue Apr 2 15:29:54 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.52600000000000002}]},{"osd":7,"last update":"Tue Apr 2 15:30:21 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.57799999999999996}]}]},{"osd":5,"up_from":39,"seq":167503724573,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5688,"kb_used_data":560,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739464,"statfs":{"total":95995035648,"available":95989211136,"internally_reserved":0,"allocated":573440,"data_stored":235726,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 15:30:34 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.39800000000000002,"5min":0.307,"15min":0.307},"max":{"1min":0.65700000000000003,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.497,"5min":0.52300000000000002,"15min":0.52700000000000002},"min":{"1min":0.36399999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.63100000000000001,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.54600000000000004}]},{"osd":1,"last update":"Tue Apr 2 15:30:34 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.53000000000000003,"15min":0.52800000000000002},"min":{"1min":0.41199999999999998,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.67900000000000005,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.55000000000000004,"15min":0.55400000000000005},"min":{"1min":0.43099999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.64800000000000002,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.40000000000000002}]},{"osd":2,"last update":"Tue Apr 2 15:30:34 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.53100000000000003,"15min":0.53700000000000003},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.61599999999999999,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.376},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56699999999999995,"15min":0.56799999999999995},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.66000000000000003,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.35099999999999998}]},{"osd":3,"last update":"Tue Apr 2 15:30:34 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.58099999999999996,"15min":0.58499999999999996},"min":{"1min":0.435,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.67800000000000005,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.46200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.69299999999999995,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.58999999999999997}]},{"osd":4,"last update":"Tue Apr 2 15:30:34 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.42099999999999999,"15min":0.42299999999999999},"min":{"1min":0.311,"5min":0.308,"15min":0.308},"max":{"1min":0.624,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.42699999999999999,"15min":0.42899999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.58899999999999997,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.52900000000000003}]},{"osd":6,"last update":"Tue Apr 2 15:29:59 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.61299999999999999}]},{"osd":7,"last update":"Tue Apr 2 15:30:21 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.57599999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-02T15:30:44.288 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph pg dump --format=json 2024-04-02T15:30:44.500 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:30:45.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:45 smithi106 ceph-mon[25843]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:45.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:45 smithi106 ceph-mon[25843]: from='client.14619 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-02T15:30:45.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:45 smithi087 ceph-mon[24900]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:45.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:45 smithi087 ceph-mon[24900]: from='client.14619 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-02T15:30:45.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:45 smithi087 ceph-mon[29707]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:45.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:45 smithi087 ceph-mon[29707]: from='client.14619 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-02T15:30:46.229 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:30:45 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:30:45] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:30:46.538 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:30:46 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:30:46] "GET /metrics HTTP/1.1" 200 191475 "" "Prometheus/2.33.4" 2024-04-02T15:30:47.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:46 smithi087 ceph-mon[29707]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:47.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:46 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:47.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:46 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:47.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:46 smithi087 ceph-mon[24900]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:47.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:46 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:47.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:46 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:47.288 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-02T15:30:47.289 INFO:teuthology.orchestra.run.smithi087.stderr:dumped all 2024-04-02T15:30:47.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:46 smithi106 ceph-mon[25843]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:47.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:46 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:47.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:46 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:47.926 INFO:teuthology.orchestra.run.smithi087.stdout:{"pg_ready":true,"pg_map":{"version":39,"stamp":"2024-04-02T15:30:47.226682+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48808,"kb_used_data":6056,"kb_used_omap":0,"kb_used_meta":42688,"kb_avail":749912408,"statfs":{"total":767960285184,"available":767910305792,"internally_reserved":0,"allocated":6201344,"data_stored":3473654,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43712512},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001822"},"pg_stats":[{"pgid":"1.0","version":"59'128","reported_seq":267,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-04-02T15:29:35.303161+0000","last_change":"2024-04-02T15:29:21.613260+0000","last_active":"2024-04-02T15:29:35.303161+0000","last_peered":"2024-04-02T15:29:35.303161+0000","last_clean":"2024-04-02T15:29:35.303161+0000","last_became_active":"2024-04-02T15:29:21.612932+0000","last_became_peered":"2024-04-02T15:29:21.612932+0000","last_unstale":"2024-04-02T15:29:35.303161+0000","last_undegraded":"2024-04-02T15:29:35.303161+0000","last_fullsized":"2024-04-02T15:29:35.303161+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-02T15:27:25.608662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-02T15:27:25.608662+0000","last_clean_scrub_stamp":"2024-04-02T15:27:25.608662+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-03T18:32:49.541893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234005,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6084,"kb_used_data":1084,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739068,"statfs":{"total":95995035648,"available":95988805632,"internally_reserved":0,"allocated":1110016,"data_stored":764977,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 15:30:20 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.41999999999999998}]},{"osd":1,"last update":"Tue Apr 2 15:30:20 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.48299999999999998}]},{"osd":2,"last update":"Tue Apr 2 15:30:20 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.51900000000000002}]},{"osd":3,"last update":"Tue Apr 2 15:30:20 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.63800000000000001}]},{"osd":4,"last update":"Tue Apr 2 15:30:20 2024","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.56000000000000005}]},{"osd":5,"last update":"Tue Apr 2 15:30:20 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.438,"15min":0.438},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.626}]},{"osd":6,"last update":"Tue Apr 2 15:30:20 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.59599999999999997}]}]},{"osd":6,"up_from":46,"seq":197568495640,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6140,"kb_used_data":1076,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739012,"statfs":{"total":95995035648,"available":95988748288,"internally_reserved":0,"allocated":1101824,"data_stored":764389,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 15:29:55 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.50900000000000001}]},{"osd":1,"last update":"Tue Apr 2 15:29:55 2024","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.495},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.41899999999999998}]},{"osd":2,"last update":"Tue Apr 2 15:29:55 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.91600000000000004,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.42499999999999999}]},{"osd":3,"last update":"Tue Apr 2 15:29:55 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.54500000000000004}]},{"osd":4,"last update":"Tue Apr 2 15:29:55 2024","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.374},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.44}]},{"osd":5,"last update":"Tue Apr 2 15:29:55 2024","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.442,"5min":0.442,"15min":0.442},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.53200000000000003}]},{"osd":7,"last update":"Tue Apr 2 15:30:18 2024","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.56200000000000006}]}]},{"osd":1,"up_from":13,"seq":55834574895,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6332,"kb_used_data":564,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738820,"statfs":{"total":95995035648,"available":95988551680,"internally_reserved":0,"allocated":577536,"data_stored":236049,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 15:30:10 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.35899999999999999,"15min":0.34100000000000003},"min":{"1min":0.221,"5min":0.22,"15min":0.22},"max":{"1min":0.624,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.374,"15min":0.36199999999999999},"min":{"1min":0.247,"5min":0.224,"15min":0.224},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.251}]},{"osd":2,"last update":"Tue Apr 2 15:30:26 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.41999999999999998,"15min":0.41799999999999998},"min":{"1min":0.28299999999999997,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.68400000000000005,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.38200000000000001,"5min":0.39400000000000002,"15min":0.39200000000000002},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.27700000000000002}]},{"osd":3,"last update":"Tue Apr 2 15:29:50 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.441,"15min":0.44700000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.48299999999999998,"15min":0.48499999999999999},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.35399999999999998}]},{"osd":4,"last update":"Tue Apr 2 15:30:10 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.55000000000000004,"15min":0.55400000000000005},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.71299999999999997,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56899999999999995,"15min":0.57099999999999995},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.57099999999999995}]},{"osd":5,"last update":"Tue Apr 2 15:30:35 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.58299999999999996,"15min":0.57899999999999996},"min":{"1min":0.39600000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.498},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.73099999999999998,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.55200000000000005}]},{"osd":6,"last update":"Tue Apr 2 15:29:55 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":18.087},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.435}]},{"osd":7,"last update":"Tue Apr 2 15:30:20 2024","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.54100000000000004}]}]},{"osd":0,"up_from":8,"seq":34359738420,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6916,"kb_used_data":1084,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738236,"statfs":{"total":95995035648,"available":95987953664,"internally_reserved":0,"allocated":1110016,"data_stored":764977,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Tue Apr 2 15:30:04 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.35899999999999999,"15min":0.34999999999999998},"min":{"1min":0.23400000000000001,"5min":0.215,"15min":0.215},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.36699999999999999,"15min":0.35099999999999998},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":0.57699999999999996,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.55600000000000005}]},{"osd":2,"last update":"Tue Apr 2 15:30:32 2024","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.38400000000000001,"15min":0.379},"min":{"1min":0.27000000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.39900000000000002,"15min":0.39400000000000002},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.52400000000000002,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.622}]},{"osd":3,"last update":"Tue Apr 2 15:29:47 2024","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.44600000000000001,"15min":0.44400000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.60199999999999998}]},{"osd":4,"last update":"Tue Apr 2 15:30:12 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.56499999999999995,"15min":0.56899999999999995},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.73899999999999999},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.60099999999999998,"15min":0.60399999999999998},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.76100000000000001,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.70699999999999996}]},{"osd":5,"last update":"Tue Apr 2 15:30:36 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.38800000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58499999999999996,"15min":0.58599999999999997},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.71599999999999997,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.72899999999999998}]},{"osd":6,"last update":"Tue Apr 2 15:29:56 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.68899999999999995}]},{"osd":7,"last update":"Tue Apr 2 15:30:23 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.68200000000000005}]}]},{"osd":2,"up_from":18,"seq":77309411370,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6200,"kb_used_data":560,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738952,"statfs":{"total":95995035648,"available":95988686848,"internally_reserved":0,"allocated":573440,"data_stored":235438,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 15:30:33 2024","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.36699999999999999,"15min":0.35799999999999998},"min":{"1min":0.24299999999999999,"5min":0.224,"15min":0.224},"max":{"1min":0.47299999999999998,"5min":0.56799999999999995,"15min":0.56799999999999995},"last":0.374},{"interface":"front","average":{"1min":0.35099999999999998,"5min":0.36599999999999999,"15min":0.36199999999999999},"min":{"1min":0.25800000000000001,"5min":0.252,"15min":0.252},"max":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.24099999999999999}]},{"osd":1,"last update":"Tue Apr 2 15:30:33 2024","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.38300000000000001,"15min":0.379},"min":{"1min":0.27900000000000003,"5min":0.248,"15min":0.248},"max":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.35299999999999998,"5min":0.371,"15min":0.36699999999999999},"min":{"1min":0.24299999999999999,"5min":0.221,"15min":0.221},"max":{"1min":0.48599999999999999,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.313}]},{"osd":3,"last update":"Tue Apr 2 15:29:50 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"min":{"1min":0.30399999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.53700000000000003,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.441,"5min":0.41999999999999998,"15min":0.41599999999999998},"min":{"1min":0.28000000000000003,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.623,"5min":0.623,"15min":0.623},"last":0.34399999999999997}]},{"osd":4,"last update":"Tue Apr 2 15:30:13 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.91400000000000003,"15min":0.98199999999999998},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.67500000000000004,"5min":8.4489999999999998,"15min":8.4489999999999998},"last":0.501},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.60299999999999998,"15min":0.61499999999999999},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.73399999999999999,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.60499999999999998}]},{"osd":5,"last update":"Tue Apr 2 15:30:41 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.55500000000000005,"15min":0.56000000000000005},"min":{"1min":0.41199999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.71899999999999997,"5min":0.747,"15min":0.747},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.57699999999999996,"15min":0.58499999999999996},"min":{"1min":0.39300000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.70499999999999996,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.48099999999999998}]},{"osd":6,"last update":"Tue Apr 2 15:29:55 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.45700000000000002}]},{"osd":7,"last update":"Tue Apr 2 15:30:19 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.48799999999999999}]}]},{"osd":3,"up_from":26,"seq":111669149734,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5692,"kb_used_data":564,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739460,"statfs":{"total":95995035648,"available":95989207040,"internally_reserved":0,"allocated":577536,"data_stored":236049,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 15:29:49 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.36199999999999999,"15min":0.35399999999999998},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.318},{"interface":"front","average":{"1min":0.41799999999999998,"5min":0.38800000000000001,"15min":0.38300000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.40500000000000003}]},{"osd":1,"last update":"Tue Apr 2 15:29:49 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.41599999999999998,"15min":0.41199999999999998},"min":{"1min":0.27900000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.435,"5min":0.41499999999999998,"15min":0.41099999999999998},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.379}]},{"osd":2,"last update":"Tue Apr 2 15:29:49 2024","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.42399999999999999,"15min":0.41599999999999998},"min":{"1min":0.315,"5min":0.308,"15min":0.308},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.434,"5min":0.42799999999999999,"15min":0.42699999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.45100000000000001}]},{"osd":4,"last update":"Tue Apr 2 15:30:15 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.58799999999999997,"15min":0.58399999999999996},"min":{"1min":0.40999999999999998,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.89700000000000002,"5min":1.0309999999999999,"15min":1.0309999999999999},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.56299999999999994,"15min":0.55500000000000005},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.59499999999999997}]},{"osd":5,"last update":"Tue Apr 2 15:30:36 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.60299999999999998,"15min":0.60399999999999998},"min":{"1min":0.41899999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.84099999999999997,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.63900000000000001}]},{"osd":6,"last update":"Tue Apr 2 15:29:55 2024","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.622},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":1.0069999999999999,"5min":1.0069999999999999,"15min":1.0069999999999999},"last":0.60299999999999998}]},{"osd":7,"last update":"Tue Apr 2 15:30:20 2024","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.65800000000000003}]}]},{"osd":4,"up_from":31,"seq":133143986209,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5756,"kb_used_data":564,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739396,"statfs":{"total":95995035648,"available":95989141504,"internally_reserved":0,"allocated":577536,"data_stored":236049,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 15:30:15 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.53000000000000003,"15min":0.53200000000000003},"min":{"1min":0.29499999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.75,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.434},{"interface":"front","average":{"1min":0.499,"5min":0.51300000000000001,"15min":0.51500000000000001},"min":{"1min":0.36699999999999999,"5min":0.307,"15min":0.307},"max":{"1min":0.68600000000000005,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.38}]},{"osd":1,"last update":"Tue Apr 2 15:30:15 2024","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.504,"15min":0.51000000000000001},"min":{"1min":0.35299999999999998,"5min":0.309,"15min":0.309},"max":{"1min":0.623,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.53600000000000003,"15min":0.54000000000000004},"min":{"1min":0.33800000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.83899999999999997,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.36799999999999999}]},{"osd":2,"last update":"Tue Apr 2 15:30:15 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.54300000000000004,"15min":0.54100000000000004},"min":{"1min":0.40500000000000003,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.80000000000000004,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.499,"5min":0.53800000000000003,"15min":0.54400000000000004},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.73099999999999998,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.39800000000000002}]},{"osd":3,"last update":"Tue Apr 2 15:30:15 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.52900000000000003,"15min":0.53600000000000003},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.72599999999999998,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.54200000000000004,"15min":0.54700000000000004},"min":{"1min":0.34899999999999998,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.72699999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.38900000000000001}]},{"osd":5,"last update":"Tue Apr 2 15:30:36 2024","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.40799999999999997,"15min":0.40000000000000002},"min":{"1min":0.309,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.65000000000000002,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.441,"15min":0.433},"min":{"1min":0.33300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.79800000000000004,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.317}]},{"osd":6,"last update":"Tue Apr 2 15:29:54 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.41399999999999998}]},{"osd":7,"last update":"Tue Apr 2 15:30:21 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.441},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.46899999999999997}]}]},{"osd":5,"up_from":39,"seq":167503724574,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5688,"kb_used_data":560,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739464,"statfs":{"total":95995035648,"available":95989211136,"internally_reserved":0,"allocated":573440,"data_stored":235726,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 15:30:34 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.39800000000000002,"5min":0.307,"15min":0.307},"max":{"1min":0.65700000000000003,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.497,"5min":0.52300000000000002,"15min":0.52700000000000002},"min":{"1min":0.36399999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.63100000000000001,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.51400000000000001}]},{"osd":1,"last update":"Tue Apr 2 15:30:34 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.53000000000000003,"15min":0.52800000000000002},"min":{"1min":0.41199999999999998,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.67900000000000005,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.55000000000000004,"15min":0.55400000000000005},"min":{"1min":0.43099999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.64800000000000002,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.54600000000000004}]},{"osd":2,"last update":"Tue Apr 2 15:30:34 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.53100000000000003,"15min":0.53700000000000003},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.61599999999999999,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56699999999999995,"15min":0.56799999999999995},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.66000000000000003,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.47399999999999998}]},{"osd":3,"last update":"Tue Apr 2 15:30:34 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.58099999999999996,"15min":0.58499999999999996},"min":{"1min":0.435,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.67800000000000005,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.46200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.69299999999999995,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.627}]},{"osd":4,"last update":"Tue Apr 2 15:30:34 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.42099999999999999,"15min":0.42299999999999999},"min":{"1min":0.311,"5min":0.308,"15min":0.308},"max":{"1min":0.624,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.42699999999999999,"15min":0.42899999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.58899999999999997,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.34399999999999997}]},{"osd":6,"last update":"Tue Apr 2 15:29:59 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.40899999999999997}]},{"osd":7,"last update":"Tue Apr 2 15:30:21 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.52200000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-02T15:30:47.927 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-02T15:30:47.927 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-02T15:30:47.927 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-02T15:30:47.927 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -- ceph health --format=json 2024-04-02T15:30:48.775 INFO:teuthology.orchestra.run.smithi087.stderr:Inferring config /var/lib/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/mon.a/config 2024-04-02T15:30:49.241 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:48 smithi087 ceph-mon[29707]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:49.241 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:48 smithi087 ceph-mon[29707]: from='client.24520 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-02T15:30:49.242 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:48 smithi087 ceph-mon[24900]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:49.242 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:48 smithi087 ceph-mon[24900]: from='client.24520 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-02T15:30:49.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:48 smithi106 ceph-mon[25843]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:49.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:48 smithi106 ceph-mon[25843]: from='client.24520 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-02T15:30:50.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:50 smithi106 ceph-mon[25843]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:50.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:50 smithi087 ceph-mon[24900]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:50.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:50 smithi087 ceph-mon[29707]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:52.465 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:52 smithi087 ceph-mon[29707]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:52.465 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:52 smithi087 ceph-mon[24900]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:52.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:52 smithi106 ceph-mon[25843]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:53.416 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-02T15:30:53.416 INFO:teuthology.orchestra.run.smithi087.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-02T15:30:54.116 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:30:53 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:30:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:30:54.151 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-02T15:30:54.151 INFO:tasks.cephadm:Setup complete, yielding 2024-04-02T15:30:54.151 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-02T15:30:54.163 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi087.front.sepia.ceph.com 2024-04-02T15:30:54.163 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'radosgw-admin realm create --rgw-realm=r --default' 2024-04-02T15:30:54.512 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:54 smithi087 ceph-mon[29707]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:54.512 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:54 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/1974562057' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-02T15:30:54.512 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:54 smithi087 ceph-mon[24900]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:54.512 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:54 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/1974562057' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-02T15:30:54.513 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:30:54 smithi087 podman[49848]: 2024-04-02 15:30:54.357025848 +0000 UTC m=+0.182378877 volume create d6f2c6158a59d7209c44b0883f36076c2f5d17df618f38317f3e6a3ecaaee91b 2024-04-02T15:30:54.513 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:30:54 smithi087 podman[49848]: 2024-04-02 15:30:54.515865966 +0000 UTC m=+0.341219037 container create 51d8db44496129a11b1da7ccfce94cc160b83eafee3e296dc6a17c607be1e561 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T15:30:54.527 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:54 smithi106 ceph-mon[25843]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:54.527 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:54 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/1974562057' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-02T15:30:55.033 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:30:54 smithi087 podman[49848]: 2024-04-02 15:30:54.785262903 +0000 UTC m=+0.610615932 container init 51d8db44496129a11b1da7ccfce94cc160b83eafee3e296dc6a17c607be1e561 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T15:30:55.033 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:30:54 smithi087 podman[49848]: 2024-04-02 15:30:54.787900415 +0000 UTC m=+0.613253451 container start 51d8db44496129a11b1da7ccfce94cc160b83eafee3e296dc6a17c607be1e561 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T15:30:55.033 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:30:54 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[49920]: level=info ts=2024-04-02T15:30:54.798Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-04-02T15:30:55.033 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:30:54 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[49920]: level=info ts=2024-04-02T15:30:54.798Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-04-02T15:30:55.033 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:30:54 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[49920]: level=info ts=2024-04-02T15:30:54.800Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.87 port=9094 2024-04-02T15:30:55.033 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:30:54 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[49920]: level=info ts=2024-04-02T15:30:54.800Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-02T15:30:55.033 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:30:54 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[49920]: level=info ts=2024-04-02T15:30:54.825Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-02T15:30:55.033 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:30:54 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[49920]: level=info ts=2024-04-02T15:30:54.825Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-02T15:30:55.034 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:30:54 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[49920]: level=info ts=2024-04-02T15:30:54.828Z caller=main.go:518 msg=Listening address=:9093 2024-04-02T15:30:55.034 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:30:54 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[49920]: level=info ts=2024-04-02T15:30:54.828Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-04-02T15:30:55.034 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:30:54 smithi087 bash[49848]: 51d8db44496129a11b1da7ccfce94cc160b83eafee3e296dc6a17c607be1e561 2024-04-02T15:30:55.034 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:30:54 smithi087 systemd[1]: Started Ceph alertmanager.a for 0400842c-f105-11ee-b647-cb9ed24678a4. 2024-04-02T15:30:56.330 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:55 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:56.330 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:55 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:56.330 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:55 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:56.330 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:55 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-02T15:30:56.330 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:55 smithi087 ceph-mon[24900]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-02T15:30:56.330 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:55 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:56.330 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:55 smithi087 ceph-mon[24900]: Deploying daemon grafana.a on smithi106 2024-04-02T15:30:56.331 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:55 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:56.331 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:55 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:56.331 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:55 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:56.331 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:55 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:56.331 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:55 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-02T15:30:56.331 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:55 smithi087 ceph-mon[29707]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-02T15:30:56.331 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:55 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:56.331 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:55 smithi087 ceph-mon[29707]: Deploying daemon grafana.a on smithi106 2024-04-02T15:30:56.331 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:55 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:56.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:55 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:56.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:55 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:56.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:55 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:56.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:55 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-02T15:30:56.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:55 smithi106 ceph-mon[25843]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-02T15:30:56.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:55 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:56.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:55 smithi106 ceph-mon[25843]: Deploying daemon grafana.a on smithi106 2024-04-02T15:30:56.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:55 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:30:56.620 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:30:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:30:56] "GET /metrics HTTP/1.1" 200 191482 "" "Prometheus/2.33.4" 2024-04-02T15:30:56.938 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:30:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[49920]: level=info ts=2024-04-02T15:30:56.801Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.00043121s 2024-04-02T15:30:57.205 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:57 smithi087 ceph-mon[24900]: pgmap v43: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:57.205 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:57 smithi087 ceph-mon[29707]: pgmap v43: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:57.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:57 smithi106 ceph-mon[25843]: pgmap v43: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:59.051 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-02T15:30:59.052 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "9bfe5281-8e8b-4ce7-adf3-af6510edad27", 2024-04-02T15:30:59.052 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "r", 2024-04-02T15:30:59.052 INFO:teuthology.orchestra.run.smithi087.stdout: "current_period": "ecf30e6e-42b9-4430-a780-b5a5eb862a9a", 2024-04-02T15:30:59.052 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 1 2024-04-02T15:30:59.052 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-02T15:30:59.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:59 smithi087 ceph-mon[29707]: pgmap v44: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:59.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:59 smithi087 ceph-mon[29707]: osdmap e60: 8 total, 8 up, 8 in 2024-04-02T15:30:59.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:59 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/1584902870' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-02T15:30:59.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:30:59 smithi087 ceph-mon[29707]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-02T15:30:59.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:59 smithi087 ceph-mon[24900]: pgmap v44: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:59.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:59 smithi087 ceph-mon[24900]: osdmap e60: 8 total, 8 up, 8 in 2024-04-02T15:30:59.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:59 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/1584902870' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-02T15:30:59.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:30:59 smithi087 ceph-mon[24900]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-02T15:30:59.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:59 smithi106 ceph-mon[25843]: pgmap v44: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:30:59.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:59 smithi106 ceph-mon[25843]: osdmap e60: 8 total, 8 up, 8 in 2024-04-02T15:30:59.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:59 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/1584902870' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-02T15:30:59.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:30:59 smithi106 ceph-mon[25843]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-02T15:30:59.715 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'radosgw-admin zonegroup create --rgw-zonegroup=default --master --default' 2024-04-02T15:31:00.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:00 smithi087 ceph-mon[29707]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-04-02T15:31:00.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:00 smithi087 ceph-mon[29707]: osdmap e61: 8 total, 8 up, 8 in 2024-04-02T15:31:00.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:00 smithi087 ceph-mon[24900]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-04-02T15:31:00.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:00 smithi087 ceph-mon[24900]: osdmap e61: 8 total, 8 up, 8 in 2024-04-02T15:31:00.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:00 smithi106 ceph-mon[25843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-04-02T15:31:00.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:00 smithi106 ceph-mon[25843]: osdmap e61: 8 total, 8 up, 8 in 2024-04-02T15:31:01.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:01 smithi106 ceph-mon[25843]: pgmap v47: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:31:01.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:01 smithi106 ceph-mon[25843]: osdmap e62: 8 total, 8 up, 8 in 2024-04-02T15:31:01.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:01 smithi087 ceph-mon[29707]: pgmap v47: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:31:01.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:01 smithi087 ceph-mon[29707]: osdmap e62: 8 total, 8 up, 8 in 2024-04-02T15:31:01.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:01 smithi087 ceph-mon[24900]: pgmap v47: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:31:01.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:01 smithi087 ceph-mon[24900]: osdmap e62: 8 total, 8 up, 8 in 2024-04-02T15:31:01.870 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:31:01 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:31:01] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:31:02.301 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-02T15:31:02.301 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "f3e98b35-7f05-4a1a-a5b0-45b7ebc272af", 2024-04-02T15:31:02.301 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "default", 2024-04-02T15:31:02.301 INFO:teuthology.orchestra.run.smithi087.stdout: "api_name": "default", 2024-04-02T15:31:02.301 INFO:teuthology.orchestra.run.smithi087.stdout: "is_master": "true", 2024-04-02T15:31:02.301 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [], 2024-04-02T15:31:02.301 INFO:teuthology.orchestra.run.smithi087.stdout: "hostnames": [], 2024-04-02T15:31:02.301 INFO:teuthology.orchestra.run.smithi087.stdout: "hostnames_s3website": [], 2024-04-02T15:31:02.302 INFO:teuthology.orchestra.run.smithi087.stdout: "master_zone": "", 2024-04-02T15:31:02.302 INFO:teuthology.orchestra.run.smithi087.stdout: "zones": [], 2024-04-02T15:31:02.302 INFO:teuthology.orchestra.run.smithi087.stdout: "placement_targets": [], 2024-04-02T15:31:02.302 INFO:teuthology.orchestra.run.smithi087.stdout: "default_placement": "", 2024-04-02T15:31:02.302 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_id": "9bfe5281-8e8b-4ce7-adf3-af6510edad27", 2024-04-02T15:31:02.302 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_policy": { 2024-04-02T15:31:02.302 INFO:teuthology.orchestra.run.smithi087.stdout: "groups": [] 2024-04-02T15:31:02.302 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-02T15:31:02.302 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-02T15:31:02.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:02 smithi106 ceph-mon[25843]: pgmap v49: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-04-02T15:31:02.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:02 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:02.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:02 smithi087 ceph-mon[29707]: pgmap v49: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-04-02T15:31:02.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:02 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:02.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:02 smithi087 ceph-mon[24900]: pgmap v49: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-04-02T15:31:02.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:02 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:02.998 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default' 2024-04-02T15:31:04.116 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:31:03 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:31:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:31:04.800 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:04 smithi087 ceph-mon[29707]: pgmap v50: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-04-02T15:31:04.801 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:04 smithi087 ceph-mon[24900]: pgmap v50: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-04-02T15:31:04.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:04 smithi106 ceph-mon[25843]: pgmap v50: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-04-02T15:31:05.114 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:31:04 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[49920]: level=info ts=2024-04-02T15:31:04.803Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002756887s 2024-04-02T15:31:05.602 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-02T15:31:05.602 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "4b1d6266-9f83-43ad-a2df-8910ecae0ec1", 2024-04-02T15:31:05.602 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "z", 2024-04-02T15:31:05.603 INFO:teuthology.orchestra.run.smithi087.stdout: "domain_root": "z.rgw.meta:root", 2024-04-02T15:31:05.603 INFO:teuthology.orchestra.run.smithi087.stdout: "control_pool": "z.rgw.control", 2024-04-02T15:31:05.603 INFO:teuthology.orchestra.run.smithi087.stdout: "gc_pool": "z.rgw.log:gc", 2024-04-02T15:31:05.603 INFO:teuthology.orchestra.run.smithi087.stdout: "lc_pool": "z.rgw.log:lc", 2024-04-02T15:31:05.603 INFO:teuthology.orchestra.run.smithi087.stdout: "log_pool": "z.rgw.log", 2024-04-02T15:31:05.603 INFO:teuthology.orchestra.run.smithi087.stdout: "intent_log_pool": "z.rgw.log:intent", 2024-04-02T15:31:05.603 INFO:teuthology.orchestra.run.smithi087.stdout: "usage_log_pool": "z.rgw.log:usage", 2024-04-02T15:31:05.603 INFO:teuthology.orchestra.run.smithi087.stdout: "roles_pool": "z.rgw.meta:roles", 2024-04-02T15:31:05.603 INFO:teuthology.orchestra.run.smithi087.stdout: "reshard_pool": "z.rgw.log:reshard", 2024-04-02T15:31:05.603 INFO:teuthology.orchestra.run.smithi087.stdout: "user_keys_pool": "z.rgw.meta:users.keys", 2024-04-02T15:31:05.603 INFO:teuthology.orchestra.run.smithi087.stdout: "user_email_pool": "z.rgw.meta:users.email", 2024-04-02T15:31:05.603 INFO:teuthology.orchestra.run.smithi087.stdout: "user_swift_pool": "z.rgw.meta:users.swift", 2024-04-02T15:31:05.603 INFO:teuthology.orchestra.run.smithi087.stdout: "user_uid_pool": "z.rgw.meta:users.uid", 2024-04-02T15:31:05.604 INFO:teuthology.orchestra.run.smithi087.stdout: "otp_pool": "z.rgw.otp", 2024-04-02T15:31:05.604 INFO:teuthology.orchestra.run.smithi087.stdout: "system_key": { 2024-04-02T15:31:05.604 INFO:teuthology.orchestra.run.smithi087.stdout: "access_key": "", 2024-04-02T15:31:05.604 INFO:teuthology.orchestra.run.smithi087.stdout: "secret_key": "" 2024-04-02T15:31:05.604 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-02T15:31:05.604 INFO:teuthology.orchestra.run.smithi087.stdout: "placement_pools": [ 2024-04-02T15:31:05.604 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-02T15:31:05.604 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "default-placement", 2024-04-02T15:31:05.604 INFO:teuthology.orchestra.run.smithi087.stdout: "val": { 2024-04-02T15:31:05.604 INFO:teuthology.orchestra.run.smithi087.stdout: "index_pool": "z.rgw.buckets.index", 2024-04-02T15:31:05.604 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_classes": { 2024-04-02T15:31:05.604 INFO:teuthology.orchestra.run.smithi087.stdout: "STANDARD": { 2024-04-02T15:31:05.604 INFO:teuthology.orchestra.run.smithi087.stdout: "data_pool": "z.rgw.buckets.data" 2024-04-02T15:31:05.605 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-02T15:31:05.605 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-02T15:31:05.605 INFO:teuthology.orchestra.run.smithi087.stdout: "data_extra_pool": "z.rgw.buckets.non-ec", 2024-04-02T15:31:05.605 INFO:teuthology.orchestra.run.smithi087.stdout: "index_type": 0 2024-04-02T15:31:05.605 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-02T15:31:05.605 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-02T15:31:05.605 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2024-04-02T15:31:05.605 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_id": "9bfe5281-8e8b-4ce7-adf3-af6510edad27", 2024-04-02T15:31:05.605 INFO:teuthology.orchestra.run.smithi087.stdout: "notif_pool": "z.rgw.log:notif" 2024-04-02T15:31:05.605 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-02T15:31:06.297 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:31:06 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:31:06] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:31:06.321 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'radosgw-admin period update --rgw-realm=r --commit' 2024-04-02T15:31:06.620 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:31:06 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:31:06] "GET /metrics HTTP/1.1" 200 192769 "" "Prometheus/2.33.4" 2024-04-02T15:31:07.003 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:31:06 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:31:06] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:31:07.349 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:07 smithi087 ceph-mon[24900]: pgmap v51: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 1 op/s 2024-04-02T15:31:07.349 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:07 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:07.349 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:07 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:07.349 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:07 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:07.349 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:07 smithi087 ceph-mon[29707]: pgmap v51: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 1 op/s 2024-04-02T15:31:07.349 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:07 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:07.349 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:07 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:07.350 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:07 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:07.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:07 smithi106 ceph-mon[25843]: pgmap v51: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 1 op/s 2024-04-02T15:31:07.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:07 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:07.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:07 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:07.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:07 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:07.620 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:31:07 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:31:07] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:31:08.072 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:07 smithi106 systemd[1]: Starting Ceph grafana.a for 0400842c-f105-11ee-b647-cb9ed24678a4... 2024-04-02T15:31:08.355 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 podman[44547]: 2024-04-02 15:31:07.977883507 +0000 UTC m=+0.017277053 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-04-02T15:31:08.355 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 podman[44547]: 2024-04-02 15:31:08.138798948 +0000 UTC m=+0.178192486 container create 8956d795b0b49a3d071492d47c4e3a589ed5347bf54c4846f33b8d2bf88840d7 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a, maintainer=Paul Cuzner , distribution-scope=public, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container, build-date=2022-03-28T10:36:18.413762, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, name=ubi8, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, vcs-type=git, version=8.5, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.buildah.version=1.24.2, release=236.1648460182, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64) 2024-04-02T15:31:08.355 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 podman[44547]: 2024-04-02 15:31:08.358793237 +0000 UTC m=+0.398186793 container init 8956d795b0b49a3d071492d47c4e3a589ed5347bf54c4846f33b8d2bf88840d7 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a, name=ubi8, com.redhat.component=ubi8-container, description=Ceph Grafana Container, vcs-type=git, io.buildah.version=1.24.2, distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, build-date=2022-03-28T10:36:18.413762, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=236.1648460182, maintainer=Paul Cuzner , vendor=Red Hat, Inc., io.openshift.tags=base rhel8) 2024-04-02T15:31:08.607 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-mon[25843]: pgmap v52: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 872 B/s rd, 1.9 KiB/s wr, 3 op/s 2024-04-02T15:31:08.607 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:08.607 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:08.607 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:08.608 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:31:08.608 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:31:08.608 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:31:08.608 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:08.609 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 podman[44547]: 2024-04-02 15:31:08.362023113 +0000 UTC m=+0.401416661 container start 8956d795b0b49a3d071492d47c4e3a589ed5347bf54c4846f33b8d2bf88840d7 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, distribution-scope=public, io.buildah.version=1.24.2, maintainer=Paul Cuzner , description=Ceph Grafana Container, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-03-28T10:36:18.413762, vendor=Red Hat, Inc., architecture=x86_64, release=236.1648460182, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, name=ubi8, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-04-02T15:31:08.609 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2024-04-02T15:31:08.609 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2024-04-02T15:31:08.609 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2024-04-02T15:31:08.609 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2024-04-02T15:31:08.609 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2024-04-02T15:31:08.609 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2024-04-02T15:31:08.609 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-04-02T15:31:08.609 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-04-02T15:31:08.610 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2024-04-02T15:31:08.610 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2024-04-02T15:31:08.610 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2024-04-02T15:31:08.610 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2024-04-02T15:31:08.610 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2024-04-02T15:31:08.610 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="App mode production" logger=settings 2024-04-02T15:31:08.610 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2024-04-02T15:31:08.610 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-04-02T15:31:08.610 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Starting DB migrations" logger=migrator 2024-04-02T15:31:08.610 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create migration_log table" 2024-04-02T15:31:08.610 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create user table" 2024-04-02T15:31:08.610 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.login" 2024-04-02T15:31:08.610 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.email" 2024-04-02T15:31:08.611 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_login - v1" 2024-04-02T15:31:08.611 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_email - v1" 2024-04-02T15:31:08.611 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table user to user_v1 - v1" 2024-04-02T15:31:08.613 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create user table v2" 2024-04-02T15:31:08.613 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_login - v2" 2024-04-02T15:31:08.614 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_email - v2" 2024-04-02T15:31:08.614 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2024-04-02T15:31:08.614 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table user_v1" 2024-04-02T15:31:08.614 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add column help_flags1 to user table" 2024-04-02T15:31:08.614 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Update user table charset" 2024-04-02T15:31:08.614 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add last_seen_at column to user" 2024-04-02T15:31:08.614 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add missing user data" 2024-04-02T15:31:08.614 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_disabled column to user" 2024-04-02T15:31:08.614 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add index user.login/user.email" 2024-04-02T15:31:08.614 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_service_account column to user" 2024-04-02T15:31:08.614 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create temp user table v1-7" 2024-04-02T15:31:08.614 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v1-7" 2024-04-02T15:31:08.614 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v1-7" 2024-04-02T15:31:08.615 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v1-7" 2024-04-02T15:31:08.615 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v1-7" 2024-04-02T15:31:08.615 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Update temp_user table charset" 2024-04-02T15:31:08.615 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_email - v1" 2024-04-02T15:31:08.615 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_org_id - v1" 2024-04-02T15:31:08.615 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_code - v1" 2024-04-02T15:31:08.615 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_status - v1" 2024-04-02T15:31:08.615 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-04-02T15:31:08.615 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create temp_user v2" 2024-04-02T15:31:08.615 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v2" 2024-04-02T15:31:08.615 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v2" 2024-04-02T15:31:08.615 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v2" 2024-04-02T15:31:08.615 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v2" 2024-04-02T15:31:08.616 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="copy temp_user v1 to v2" 2024-04-02T15:31:08.616 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="drop temp_user_tmp_qwerty" 2024-04-02T15:31:08.616 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Set created for temp users that will otherwise prematurely expire" 2024-04-02T15:31:08.616 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create star table" 2024-04-02T15:31:08.616 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index star.user_id_dashboard_id" 2024-04-02T15:31:08.616 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create org table v1" 2024-04-02T15:31:08.616 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_name - v1" 2024-04-02T15:31:08.616 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create org_user table v1" 2024-04-02T15:31:08.616 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_org_id - v1" 2024-04-02T15:31:08.616 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_user_org_id_user_id - v1" 2024-04-02T15:31:08.616 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_user_id - v1" 2024-04-02T15:31:08.616 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Update org table charset" 2024-04-02T15:31:08.616 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Update org_user table charset" 2024-04-02T15:31:08.617 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate all Read Only Viewers to Viewers" 2024-04-02T15:31:08.617 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard table" 2024-04-02T15:31:08.617 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard.account_id" 2024-04-02T15:31:08.617 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_account_id_slug" 2024-04-02T15:31:08.617 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_tag table" 2024-04-02T15:31:08.617 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_tag.dasboard_id_term" 2024-04-02T15:31:08.617 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-04-02T15:31:08.617 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard to dashboard_v1 - v1" 2024-04-02T15:31:08.617 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard v2" 2024-04-02T15:31:08.617 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_org_id - v2" 2024-04-02T15:31:08.617 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_org_id_slug - v2" 2024-04-02T15:31:08.617 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 bash[44547]: 8956d795b0b49a3d071492d47c4e3a589ed5347bf54c4846f33b8d2bf88840d7 2024-04-02T15:31:08.617 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard v1 to v2" 2024-04-02T15:31:08.617 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 systemd[1]: Started Ceph grafana.a for 0400842c-f105-11ee-b647-cb9ed24678a4. 2024-04-02T15:31:08.617 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_v1" 2024-04-02T15:31:08.618 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard.data to mediumtext v1" 2024-04-02T15:31:08.618 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add column updated_by in dashboard - v2" 2024-04-02T15:31:08.618 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add column created_by in dashboard - v2" 2024-04-02T15:31:08.618 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add column gnetId in dashboard" 2024-04-02T15:31:08.619 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for gnetId in dashboard" 2024-04-02T15:31:08.619 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_id in dashboard" 2024-04-02T15:31:08.619 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for plugin_id in dashboard" 2024-04-02T15:31:08.619 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_id in dashboard_tag" 2024-04-02T15:31:08.619 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard table charset" 2024-04-02T15:31:08.619 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_tag table charset" 2024-04-02T15:31:08.619 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add column folder_id in dashboard" 2024-04-02T15:31:08.619 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add column isFolder in dashboard" 2024-04-02T15:31:08.619 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add column has_acl in dashboard" 2024-04-02T15:31:08.619 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in dashboard" 2024-04-02T15:31:08.621 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in dashboard" 2024-04-02T15:31:08.621 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index dashboard_org_id_uid" 2024-04-02T15:31:08.621 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_slug" 2024-04-02T15:31:08.621 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard title length" 2024-04-02T15:31:08.621 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index for dashboard_org_id_title_folder_id" 2024-04-02T15:31:08.622 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning" 2024-04-02T15:31:08.622 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-04-02T15:31:08.622 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning v2" 2024-04-02T15:31:08.622 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-04-02T15:31:08.622 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-04-02T15:31:08.622 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard_provisioning v1 to v2" 2024-04-02T15:31:08.622 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="drop dashboard_provisioning_tmp_qwerty" 2024-04-02T15:31:08.622 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add check_sum column" 2024-04-02T15:31:08.622 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_title" 2024-04-02T15:31:08.622 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="delete tags for deleted dashboards" 2024-04-02T15:31:08.622 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="delete stars for deleted dashboards" 2024-04-02T15:31:08.622 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_is_folder" 2024-04-02T15:31:08.622 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table" 2024-04-02T15:31:08.622 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index data_source.account_id" 2024-04-02T15:31:08.622 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index data_source.account_id_name" 2024-04-02T15:31:08.623 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_data_source_account_id - v1" 2024-04-02T15:31:08.623 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_data_source_account_id_name - v1" 2024-04-02T15:31:08.623 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table data_source to data_source_v1 - v1" 2024-04-02T15:31:08.623 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table v2" 2024-04-02T15:31:08.623 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_data_source_org_id - v2" 2024-04-02T15:31:08.623 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_data_source_org_id_name - v2" 2024-04-02T15:31:08.623 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2024-04-02T15:31:08.623 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table data_source_v1 #2" 2024-04-02T15:31:08.623 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add column with_credentials" 2024-04-02T15:31:08.623 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add secure json data column" 2024-04-02T15:31:08.623 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Update data_source table charset" 2024-04-02T15:31:08.623 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Update initial version to 1" 2024-04-02T15:31:08.623 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add read_only data column" 2024-04-02T15:31:08.623 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate logging ds to loki ds" 2024-04-02T15:31:08.623 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Update json_data with nulls" 2024-04-02T15:31:08.624 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add uid column" 2024-04-02T15:31:08.624 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid value" 2024-04-02T15:31:08.624 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index datasource_org_id_uid" 2024-04-02T15:31:08.624 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index datasource_org_id_is_default" 2024-04-02T15:31:08.624 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table" 2024-04-02T15:31:08.624 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id" 2024-04-02T15:31:08.624 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.key" 2024-04-02T15:31:08.624 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id_name" 2024-04-02T15:31:08.624 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_api_key_account_id - v1" 2024-04-02T15:31:08.624 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_key - v1" 2024-04-02T15:31:08.624 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_account_id_name - v1" 2024-04-02T15:31:08.624 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table api_key to api_key_v1 - v1" 2024-04-02T15:31:08.624 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table v2" 2024-04-02T15:31:08.624 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_api_key_org_id - v2" 2024-04-02T15:31:08.624 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_key - v2" 2024-04-02T15:31:08.624 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_org_id_name - v2" 2024-04-02T15:31:08.625 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="copy api_key v1 to v2" 2024-04-02T15:31:08.625 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table api_key_v1" 2024-04-02T15:31:08.625 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Update api_key table charset" 2024-04-02T15:31:08.625 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add expires to api_key table" 2024-04-02T15:31:08.625 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add service account foreign key" 2024-04-02T15:31:08.625 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v4" 2024-04-02T15:31:08.625 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_snapshot_v4 #1" 2024-04-02T15:31:08.625 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v5 #2" 2024-04-02T15:31:08.625 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_key - v5" 2024-04-02T15:31:08.625 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-04-02T15:31:08.625 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_snapshot_user_id - v5" 2024-04-02T15:31:08.625 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_snapshot to mediumtext v2" 2024-04-02T15:31:08.625 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_snapshot table charset" 2024-04-02T15:31:08.625 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external_delete_url to dashboard_snapshots table" 2024-04-02T15:31:08.625 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add encrypted dashboard json column" 2024-04-02T15:31:08.626 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-04-02T15:31:08.626 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create quota table v1" 2024-04-02T15:31:08.626 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_quota_org_id_user_id_target - v1" 2024-04-02T15:31:08.626 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Update quota table charset" 2024-04-02T15:31:08.626 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create plugin_setting table" 2024-04-02T15:31:08.626 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-04-02T15:31:08.626 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_version to plugin_settings" 2024-04-02T15:31:08.626 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Update plugin_setting table charset" 2024-04-02T15:31:08.626 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create session table" 2024-04-02T15:31:08.626 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist table" 2024-04-02T15:31:08.626 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist_item table" 2024-04-02T15:31:08.626 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist table v2" 2024-04-02T15:31:08.626 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist item table v2" 2024-04-02T15:31:08.626 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist table charset" 2024-04-02T15:31:08.626 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist_item table charset" 2024-04-02T15:31:08.626 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v2" 2024-04-02T15:31:08.627 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v3" 2024-04-02T15:31:08.627 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create preferences table v3" 2024-04-02T15:31:08.859 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Update preferences table charset" 2024-04-02T15:31:08.860 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add column team_id in preferences" 2024-04-02T15:31:08.860 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Update team_id column values in preferences" 2024-04-02T15:31:08.860 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add column week_start in preferences" 2024-04-02T15:31:08.860 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create alert table v1" 2024-04-02T15:31:08.860 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert org_id & id " 2024-04-02T15:31:08.860 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert state" 2024-04-02T15:31:08.860 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert dashboard_id" 2024-04-02T15:31:08.860 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v1" 2024-04-02T15:31:08.860 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-04-02T15:31:08.860 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-04-02T15:31:08.860 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-04-02T15:31:08.860 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v2" 2024-04-02T15:31:08.861 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-04-02T15:31:08.861 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="copy alert_rule_tag v1 to v2" 2024-04-02T15:31:08.861 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="drop table alert_rule_tag_v1" 2024-04-02T15:31:08.861 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification table v1" 2024-04-02T15:31:08.861 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add column is_default" 2024-04-02T15:31:08.861 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2024-04-02T15:31:08.861 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2024-04-02T15:31:08.861 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2024-04-02T15:31:08.861 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2024-04-02T15:31:08.861 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2024-04-02T15:31:08.861 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2024-04-02T15:31:08.861 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2024-04-02T15:31:08.861 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2024-04-02T15:31:08.861 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2024-04-02T15:31:08.861 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2024-04-02T15:31:08.861 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-04-02T15:31:08.862 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2024-04-02T15:31:08.862 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2024-04-02T15:31:08.862 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2024-04-02T15:31:08.862 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2024-04-02T15:31:08.862 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2024-04-02T15:31:08.862 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add column secure_settings in alert_notification" 2024-04-02T15:31:08.862 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert.settings to mediumtext" 2024-04-02T15:31:08.862 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_notification_state_alert_id" 2024-04-02T15:31:08.862 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_rule_tag_alert_id" 2024-04-02T15:31:08.862 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2024-04-02T15:31:08.864 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2024-04-02T15:31:08.864 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2024-04-02T15:31:08.864 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2024-04-02T15:31:08.864 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2024-04-02T15:31:08.864 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2024-04-02T15:31:08.864 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2024-04-02T15:31:08.864 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2024-04-02T15:31:08.864 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2024-04-02T15:31:08.864 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2024-04-02T15:31:08.864 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2024-04-02T15:31:08.864 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2024-04-02T15:31:08.864 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2024-04-02T15:31:08.864 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-04-02T15:31:08.865 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-04-02T15:31:08.865 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v3" 2024-04-02T15:31:08.865 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-04-02T15:31:08.865 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="copy annotation_tag v2 to v3" 2024-04-02T15:31:08.865 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="drop table annotation_tag_v2" 2024-04-02T15:31:08.865 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2024-04-02T15:31:08.865 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2024-04-02T15:31:08.865 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2024-04-02T15:31:08.865 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2024-04-02T15:31:08.865 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2024-04-02T15:31:08.865 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2024-04-02T15:31:08.865 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2024-04-02T15:31:08.865 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2024-04-02T15:31:08.865 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2024-04-02T15:31:08.866 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2024-04-02T15:31:08.866 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2024-04-02T15:31:08.866 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-04-02T15:31:08.866 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-04-02T15:31:08.866 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2024-04-02T15:31:08.866 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2024-04-02T15:31:08.866 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2024-04-02T15:31:08.866 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2024-04-02T15:31:08.866 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2024-04-02T15:31:08.866 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2024-04-02T15:31:08.866 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-04-02T15:31:08.866 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2024-04-02T15:31:08.866 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2024-04-02T15:31:08.866 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2024-04-02T15:31:08.867 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2024-04-02T15:31:08.867 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2024-04-02T15:31:08.867 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2024-04-02T15:31:08.867 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2024-04-02T15:31:08.867 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2024-04-02T15:31:08.867 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2024-04-02T15:31:08.867 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.team_id" 2024-04-02T15:31:08.867 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2024-04-02T15:31:08.867 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2024-04-02T15:31:08.867 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2024-04-02T15:31:08.867 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2024-04-02T15:31:08.867 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2024-04-02T15:31:08.867 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2024-04-02T15:31:08.867 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2024-04-02T15:31:08.867 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_user_id" 2024-04-02T15:31:08.868 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_team_id" 2024-04-02T15:31:08.868 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_org_id_role" 2024-04-02T15:31:08.868 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_permission" 2024-04-02T15:31:08.868 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2024-04-02T15:31:08.868 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="delete acl rules for deleted dashboards and folders" 2024-04-02T15:31:08.868 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2024-04-02T15:31:08.868 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2024-04-02T15:31:08.868 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2024-04-02T15:31:08.869 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2024-04-02T15:31:08.869 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2024-04-02T15:31:08.869 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-04-02T15:31:08.869 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2024-04-02T15:31:08.869 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2024-04-02T15:31:08.869 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2024-04-02T15:31:08.869 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2024-04-02T15:31:08.869 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2024-04-02T15:31:08.869 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-04-02T15:31:08.870 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2024-04-02T15:31:08.870 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2024-04-02T15:31:08.870 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2024-04-02T15:31:08.870 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2024-04-02T15:31:08.870 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2024-04-02T15:31:08.870 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2024-04-02T15:31:08.870 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2024-04-02T15:31:08.870 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2024-04-02T15:31:08.870 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2024-04-02T15:31:08.870 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2024-04-02T15:31:08.871 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2024-04-02T15:31:08.871 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_auth_token.user_id" 2024-04-02T15:31:08.871 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add revoked_at to the user auth token" 2024-04-02T15:31:08.871 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2024-04-02T15:31:08.871 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2024-04-02T15:31:08.871 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create short_url table v1" 2024-04-02T15:31:08.871 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index short_url.org_id-uid" 2024-04-02T15:31:08.871 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition table" 2024-04-02T15:31:08.871 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition table" 2024-04-02T15:31:08.871 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and title columns" 2024-04-02T15:31:08.871 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and uid columns" 2024-04-02T15:31:08.871 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition table data column to mediumtext in mysql" 2024-04-02T15:31:08.871 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and title columns" 2024-04-02T15:31:08.871 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and uid columns" 2024-04-02T15:31:08.871 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and title columns" 2024-04-02T15:31:08.872 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and uid columns" 2024-04-02T15:31:08.872 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add column paused in alert_definition" 2024-04-02T15:31:08.872 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition table" 2024-04-02T15:31:08.872 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition_version table" 2024-04-02T15:31:08.872 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition_version table" 2024-04-02T15:31:08.872 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-04-02T15:31:08.872 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-04-02T15:31:08.872 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition_version table data column to mediumtext in mysql" 2024-04-02T15:31:08.872 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition_version table" 2024-04-02T15:31:08.872 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_instance table" 2024-04-02T15:31:08.872 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-04-02T15:31:08.872 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, current_state columns" 2024-04-02T15:31:08.872 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add column current_state_end to alert_instance" 2024-04-02T15:31:08.872 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-04-02T15:31:08.872 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, current_state on alert_instance" 2024-04-02T15:31:08.873 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_org_id to rule_org_id in alert_instance" 2024-04-02T15:31:08.873 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_uid to rule_uid in alert_instance" 2024-04-02T15:31:08.873 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-04-02T15:31:08.873 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, current_state on alert_instance" 2024-04-02T15:31:08.873 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule table" 2024-04-02T15:31:08.873 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and title columns" 2024-04-02T15:31:08.873 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and uid columns" 2024-04-02T15:31:08.873 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-04-02T15:31:08.873 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule table data column to mediumtext in mysql" 2024-04-02T15:31:08.873 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule" 2024-04-02T15:31:08.873 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule" 2024-04-02T15:31:08.873 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule" 2024-04-02T15:31:08.873 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="remove unique index from alert_rule on org_id, title columns" 2024-04-02T15:31:08.873 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-04-02T15:31:08.874 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add dashboard_uid column to alert_rule" 2024-04-02T15:31:08.874 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add panel_id column to alert_rule" 2024-04-02T15:31:08.874 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-04-02T15:31:08.874 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule_version table" 2024-04-02T15:31:08.874 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-04-02T15:31:08.874 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-04-02T15:31:08.874 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule_version table data column to mediumtext in mysql" 2024-04-02T15:31:08.874 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule_version" 2024-04-02T15:31:08.874 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule_version" 2024-04-02T15:31:08.874 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule_version" 2024-04-02T15:31:08.874 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id=create_alert_configuration_table 2024-04-02T15:31:08.875 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add column default in alert_configuration" 2024-04-02T15:31:08.875 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-04-02T15:31:08.875 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add column org_id in alert_configuration" 2024-04-02T15:31:08.875 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_configuration table on org_id column" 2024-04-02T15:31:08.875 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id=create_ngalert_configuration_table 2024-04-02T15:31:08.875 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index in ngalert_configuration on org_id column" 2024-04-02T15:31:08.875 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="clear migration entry \"remove unified alerting data\"" 2024-04-02T15:31:08.875 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="move dashboard alerts to unified alerting" 2024-04-02T15:31:08.875 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element table v1" 2024-04-02T15:31:08.875 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element org_id-folder_id-name-kind" 2024-04-02T15:31:08.875 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element_connection table v1" 2024-04-02T15:31:08.875 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element_connection element_id-kind-connection_id" 2024-04-02T15:31:08.875 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index library_element org_id_uid" 2024-04-02T15:31:08.876 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="clone move dashboard alerts to unified alerting" 2024-04-02T15:31:08.876 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create data_keys table" 2024-04-02T15:31:08.876 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create kv_store table v1" 2024-04-02T15:31:08.876 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index kv_store.org_id-namespace-key" 2024-04-02T15:31:08.876 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="update dashboard_uid and panel_id from existing annotations" 2024-04-02T15:31:08.876 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create permission table" 2024-04-02T15:31:08.876 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index permission.role_id" 2024-04-02T15:31:08.876 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_id_action_scope" 2024-04-02T15:31:08.876 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create role table" 2024-04-02T15:31:08.876 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add column display_name" 2024-04-02T15:31:08.876 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add column group_name" 2024-04-02T15:31:08.876 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index role.org_id" 2024-04-02T15:31:08.876 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_org_id_name" 2024-04-02T15:31:08.876 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index role_org_id_uid" 2024-04-02T15:31:08.878 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:08 smithi087 ceph-mon[29707]: pgmap v52: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 872 B/s rd, 1.9 KiB/s wr, 3 op/s 2024-04-02T15:31:08.878 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:08 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:08.878 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:08 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:08.878 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:08 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:08.879 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:08 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:31:08.879 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:08 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:31:08.879 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:08 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:31:08.879 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:08 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:08.879 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:08 smithi087 ceph-mon[24900]: pgmap v52: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 872 B/s rd, 1.9 KiB/s wr, 3 op/s 2024-04-02T15:31:08.879 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:08 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:08.879 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:08 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:08.879 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:08 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:08.879 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:08 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:31:08.879 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:08 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:31:08.880 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:08 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:31:08.880 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:08 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:09.117 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create team role table" 2024-04-02T15:31:09.117 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.org_id" 2024-04-02T15:31:09.117 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_role_org_id_team_id_role_id" 2024-04-02T15:31:09.117 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.team_id" 2024-04-02T15:31:09.117 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create user role table" 2024-04-02T15:31:09.117 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.org_id" 2024-04-02T15:31:09.117 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_role_org_id_user_id_role_id" 2024-04-02T15:31:09.117 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.user_id" 2024-04-02T15:31:09.117 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create builtin role table" 2024-04-02T15:31:09.117 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.role_id" 2024-04-02T15:31:09.117 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.name" 2024-04-02T15:31:09.117 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Add column org_id to builtin_role table" 2024-04-02T15:31:09.117 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.org_id" 2024-04-02T15:31:09.118 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_org_id_role_id_role" 2024-04-02T15:31:09.118 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index role_org_id_uid" 2024-04-02T15:31:09.118 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role.uid" 2024-04-02T15:31:09.118 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="create seed assignment table" 2024-04-02T15:31:09.118 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_role_name" 2024-04-02T15:31:09.118 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="migrations completed" logger=migrator performed=381 skipped=0 duration=516.92614ms 2024-04-02T15:31:09.118 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Created default organization" logger=sqlstore 2024-04-02T15:31:09.118 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2024-04-02T15:31:09.118 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2024-04-02T15:31:09.118 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2024-04-02T15:31:09.118 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2024-04-02T15:31:09.118 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:08+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2024-04-02T15:31:09.118 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:09 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:09+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-04-02T15:31:09.118 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:09 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:09+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2024-04-02T15:31:09.119 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:09 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:09+0000 lvl=info msg="warming cache for startup" logger=ngalert 2024-04-02T15:31:09.119 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:09 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:09+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2024-04-02T15:31:09.119 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:31:09 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:31:09+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-04-02T15:31:10.298 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:10 smithi106 ceph-mon[25843]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-02T15:31:10.298 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:10 smithi106 ceph-mon[25843]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-02T15:31:10.298 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:10 smithi106 ceph-mon[25843]: Reconfiguring daemon alertmanager.a on smithi087 2024-04-02T15:31:10.298 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:10 smithi106 ceph-mon[25843]: osdmap e63: 8 total, 8 up, 8 in 2024-04-02T15:31:10.298 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:10 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/3226153813' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-02T15:31:10.298 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:10 smithi106 ceph-mon[25843]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-02T15:31:10.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:10 smithi087 ceph-mon[29707]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-02T15:31:10.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:10 smithi087 ceph-mon[29707]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-02T15:31:10.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:10 smithi087 ceph-mon[29707]: Reconfiguring daemon alertmanager.a on smithi087 2024-04-02T15:31:10.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:10 smithi087 ceph-mon[29707]: osdmap e63: 8 total, 8 up, 8 in 2024-04-02T15:31:10.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:10 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/3226153813' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-02T15:31:10.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:10 smithi087 ceph-mon[29707]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-02T15:31:10.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:10 smithi087 ceph-mon[24900]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-02T15:31:10.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:10 smithi087 ceph-mon[24900]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-02T15:31:10.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:10 smithi087 ceph-mon[24900]: Reconfiguring daemon alertmanager.a on smithi087 2024-04-02T15:31:10.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:10 smithi087 ceph-mon[24900]: osdmap e63: 8 total, 8 up, 8 in 2024-04-02T15:31:10.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:10 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/3226153813' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-02T15:31:10.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:10 smithi087 ceph-mon[24900]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-02T15:31:11.316 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:11 smithi087 ceph-mon[29707]: pgmap v54: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 778 B/s rd, 1.7 KiB/s wr, 3 op/s 2024-04-02T15:31:11.316 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:11 smithi087 ceph-mon[29707]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-04-02T15:31:11.316 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:11 smithi087 ceph-mon[29707]: osdmap e64: 8 total, 8 up, 8 in 2024-04-02T15:31:11.317 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:11 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:11.317 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:11 smithi087 ceph-mon[24900]: pgmap v54: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 778 B/s rd, 1.7 KiB/s wr, 3 op/s 2024-04-02T15:31:11.317 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:11 smithi087 ceph-mon[24900]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-04-02T15:31:11.317 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:11 smithi087 ceph-mon[24900]: osdmap e64: 8 total, 8 up, 8 in 2024-04-02T15:31:11.317 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:11 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:11.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:11 smithi106 ceph-mon[25843]: pgmap v54: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 778 B/s rd, 1.7 KiB/s wr, 3 op/s 2024-04-02T15:31:11.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:11 smithi106 ceph-mon[25843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-04-02T15:31:11.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:11 smithi106 ceph-mon[25843]: osdmap e64: 8 total, 8 up, 8 in 2024-04-02T15:31:11.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:11 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:11.587 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:31:11 smithi087 systemd[1]: Stopping Ceph alertmanager.a for 0400842c-f105-11ee-b647-cb9ed24678a4... 2024-04-02T15:31:11.587 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:31:11 smithi087 bash[51402]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager.a" found: no such container 2024-04-02T15:31:11.870 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:31:11 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[49920]: level=info ts=2024-04-02T15:31:11.590Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-04-02T15:31:11.870 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:31:11 smithi087 podman[51414]: 2024-04-02 15:31:11.621756645 +0000 UTC m=+0.124634932 container died 51d8db44496129a11b1da7ccfce94cc160b83eafee3e296dc6a17c607be1e561 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T15:31:12.254 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:12 smithi087 ceph-mon[29707]: osdmap e65: 8 total, 8 up, 8 in 2024-04-02T15:31:12.254 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:12 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/3226153813' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-02T15:31:12.254 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:12 smithi087 ceph-mon[29707]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-02T15:31:12.254 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:31:12 smithi087 podman[51414]: 2024-04-02 15:31:12.015199655 +0000 UTC m=+0.518077937 container cleanup 51d8db44496129a11b1da7ccfce94cc160b83eafee3e296dc6a17c607be1e561 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T15:31:12.255 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:31:12 smithi087 bash[51414]: ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a 2024-04-02T15:31:12.255 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:31:12 smithi087 bash[51440]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager.a" found: no such container 2024-04-02T15:31:12.255 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:12 smithi087 ceph-mon[24900]: osdmap e65: 8 total, 8 up, 8 in 2024-04-02T15:31:12.255 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:12 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/3226153813' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-02T15:31:12.255 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:12 smithi087 ceph-mon[24900]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-02T15:31:12.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:12 smithi106 ceph-mon[25843]: osdmap e65: 8 total, 8 up, 8 in 2024-04-02T15:31:12.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:12 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/3226153813' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-02T15:31:12.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:12 smithi106 ceph-mon[25843]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-02T15:31:12.583 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:31:12 smithi087 podman[51427]: 2024-04-02 15:31:12.257950055 +0000 UTC m=+0.632104596 container remove 51d8db44496129a11b1da7ccfce94cc160b83eafee3e296dc6a17c607be1e561 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T15:31:12.583 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:31:12 smithi087 podman[51427]: 2024-04-02 15:31:12.369278348 +0000 UTC m=+0.743432880 volume remove d6f2c6158a59d7209c44b0883f36076c2f5d17df618f38317f3e6a3ecaaee91b 2024-04-02T15:31:12.583 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:31:12 smithi087 systemd[1]: ceph-0400842c-f105-11ee-b647-cb9ed24678a4@alertmanager.a.service: Deactivated successfully. 2024-04-02T15:31:12.583 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:31:12 smithi087 systemd[1]: Stopped Ceph alertmanager.a for 0400842c-f105-11ee-b647-cb9ed24678a4. 2024-04-02T15:31:12.583 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:31:12 smithi087 systemd[1]: Starting Ceph alertmanager.a for 0400842c-f105-11ee-b647-cb9ed24678a4... 2024-04-02T15:31:12.864 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:31:12 smithi087 podman[51517]: 2024-04-02 15:31:12.586702037 +0000 UTC m=+0.105736730 volume create d5294a5ebd92cd81a4bb082fd9cc5b81138b215e075c59964f01a5236da7f21d 2024-04-02T15:31:12.864 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:31:12 smithi087 podman[51517]: 2024-04-02 15:31:12.500756634 +0000 UTC m=+0.019791284 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-04-02T15:31:12.864 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:31:12 smithi087 podman[51517]: 2024-04-02 15:31:12.678658718 +0000 UTC m=+0.197693373 container create 93850beda829b99b2da03b4f505a6f8d0406efd427d99369d60be5fa932fc328 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T15:31:13.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:13 smithi106 ceph-mon[25843]: pgmap v57: 97 pgs: 32 creating+peering, 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.3 KiB/s wr, 6 op/s 2024-04-02T15:31:13.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:13 smithi106 ceph-mon[25843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-04-02T15:31:13.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:13 smithi106 ceph-mon[25843]: osdmap e66: 8 total, 8 up, 8 in 2024-04-02T15:31:13.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:13 smithi087 ceph-mon[24900]: pgmap v57: 97 pgs: 32 creating+peering, 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.3 KiB/s wr, 6 op/s 2024-04-02T15:31:13.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:13 smithi087 ceph-mon[24900]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-04-02T15:31:13.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:13 smithi087 ceph-mon[24900]: osdmap e66: 8 total, 8 up, 8 in 2024-04-02T15:31:13.371 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:31:13 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:31:13] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:31:13.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:13 smithi087 ceph-mon[29707]: pgmap v57: 97 pgs: 32 creating+peering, 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.3 KiB/s wr, 6 op/s 2024-04-02T15:31:13.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:13 smithi087 ceph-mon[29707]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-04-02T15:31:13.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:13 smithi087 ceph-mon[29707]: osdmap e66: 8 total, 8 up, 8 in 2024-04-02T15:31:13.372 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:31:12 smithi087 podman[51517]: 2024-04-02 15:31:12.99785179 +0000 UTC m=+0.516886439 container init 93850beda829b99b2da03b4f505a6f8d0406efd427d99369d60be5fa932fc328 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T15:31:13.372 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:31:13 smithi087 podman[51517]: 2024-04-02 15:31:13.00072844 +0000 UTC m=+0.519763094 container start 93850beda829b99b2da03b4f505a6f8d0406efd427d99369d60be5fa932fc328 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T15:31:13.372 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:31:13 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=info ts=2024-04-02T15:31:13.010Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-04-02T15:31:13.372 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:31:13 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=info ts=2024-04-02T15:31:13.010Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-04-02T15:31:13.372 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:31:13 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=info ts=2024-04-02T15:31:13.011Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.87 port=9094 2024-04-02T15:31:13.372 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:31:13 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=info ts=2024-04-02T15:31:13.012Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-02T15:31:13.372 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:31:13 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=info ts=2024-04-02T15:31:13.037Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-02T15:31:13.372 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:31:13 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=info ts=2024-04-02T15:31:13.037Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-02T15:31:13.372 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:31:13 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=info ts=2024-04-02T15:31:13.040Z caller=main.go:518 msg=Listening address=:9093 2024-04-02T15:31:13.372 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:31:13 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=info ts=2024-04-02T15:31:13.040Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-04-02T15:31:13.372 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:31:13 smithi087 bash[51517]: 93850beda829b99b2da03b4f505a6f8d0406efd427d99369d60be5fa932fc328 2024-04-02T15:31:13.372 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:31:13 smithi087 systemd[1]: Started Ceph alertmanager.a for 0400842c-f105-11ee-b647-cb9ed24678a4. 2024-04-02T15:31:13.866 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:31:13 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:31:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:31:14.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:14 smithi106 ceph-mon[25843]: osdmap e67: 8 total, 8 up, 8 in 2024-04-02T15:31:14.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:14 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/3391646244' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-02T15:31:14.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:14 smithi106 ceph-mon[25843]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-02T15:31:14.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:14 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:14.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:14 smithi106 ceph-mon[25843]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-02T15:31:14.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:14 smithi106 ceph-mon[25843]: Reconfiguring daemon prometheus.a on smithi106 2024-04-02T15:31:14.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:14 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:14.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:14 smithi106 ceph-mon[25843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-04-02T15:31:14.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:14 smithi106 ceph-mon[25843]: osdmap e68: 8 total, 8 up, 8 in 2024-04-02T15:31:14.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:14 smithi087 ceph-mon[29707]: osdmap e67: 8 total, 8 up, 8 in 2024-04-02T15:31:14.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:14 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/3391646244' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-02T15:31:14.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:14 smithi087 ceph-mon[29707]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-02T15:31:14.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:14 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:14.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:14 smithi087 ceph-mon[29707]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-02T15:31:14.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:14 smithi087 ceph-mon[29707]: Reconfiguring daemon prometheus.a on smithi106 2024-04-02T15:31:14.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:14 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:14.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:14 smithi087 ceph-mon[29707]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-04-02T15:31:14.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:14 smithi087 ceph-mon[29707]: osdmap e68: 8 total, 8 up, 8 in 2024-04-02T15:31:14.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:14 smithi087 ceph-mon[24900]: osdmap e67: 8 total, 8 up, 8 in 2024-04-02T15:31:14.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:14 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/3391646244' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-02T15:31:14.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:14 smithi087 ceph-mon[24900]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-02T15:31:14.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:14 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:14.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:14 smithi087 ceph-mon[24900]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-02T15:31:14.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:14 smithi087 ceph-mon[24900]: Reconfiguring daemon prometheus.a on smithi106 2024-04-02T15:31:14.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:14 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:14.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:14 smithi087 ceph-mon[24900]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-04-02T15:31:14.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:14 smithi087 ceph-mon[24900]: osdmap e68: 8 total, 8 up, 8 in 2024-04-02T15:31:15.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:15 smithi106 ceph-mon[25843]: pgmap v60: 129 pgs: 32 unknown, 32 creating+peering, 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 5.7 KiB/s rd, 5 op/s 2024-04-02T15:31:15.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:15 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/3391646244' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-02T15:31:15.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:15 smithi106 ceph-mon[25843]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-02T15:31:15.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:15 smithi087 ceph-mon[24900]: pgmap v60: 129 pgs: 32 unknown, 32 creating+peering, 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 5.7 KiB/s rd, 5 op/s 2024-04-02T15:31:15.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:15 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/3391646244' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-02T15:31:15.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:15 smithi087 ceph-mon[24900]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-02T15:31:15.374 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:15 smithi087 ceph-mon[29707]: pgmap v60: 129 pgs: 32 unknown, 32 creating+peering, 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 5.7 KiB/s rd, 5 op/s 2024-04-02T15:31:15.374 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:15 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/3391646244' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-02T15:31:15.374 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:15 smithi087 ceph-mon[29707]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-02T15:31:15.374 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:31:15 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=info ts=2024-04-02T15:31:15.013Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000632875s 2024-04-02T15:31:16.168 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-02T15:31:16.169 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "588035f1-1c2b-4a40-b701-6207a4145ade", 2024-04-02T15:31:16.169 INFO:teuthology.orchestra.run.smithi087.stdout: "epoch": 1, 2024-04-02T15:31:16.169 INFO:teuthology.orchestra.run.smithi087.stdout: "predecessor_uuid": "ecf30e6e-42b9-4430-a780-b5a5eb862a9a", 2024-04-02T15:31:16.169 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_status": [], 2024-04-02T15:31:16.169 INFO:teuthology.orchestra.run.smithi087.stdout: "period_map": { 2024-04-02T15:31:16.169 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "588035f1-1c2b-4a40-b701-6207a4145ade", 2024-04-02T15:31:16.169 INFO:teuthology.orchestra.run.smithi087.stdout: "zonegroups": [ 2024-04-02T15:31:16.169 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-02T15:31:16.169 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "f3e98b35-7f05-4a1a-a5b0-45b7ebc272af", 2024-04-02T15:31:16.169 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "default", 2024-04-02T15:31:16.169 INFO:teuthology.orchestra.run.smithi087.stdout: "api_name": "default", 2024-04-02T15:31:16.169 INFO:teuthology.orchestra.run.smithi087.stdout: "is_master": "true", 2024-04-02T15:31:16.170 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [], 2024-04-02T15:31:16.170 INFO:teuthology.orchestra.run.smithi087.stdout: "hostnames": [], 2024-04-02T15:31:16.170 INFO:teuthology.orchestra.run.smithi087.stdout: "hostnames_s3website": [], 2024-04-02T15:31:16.170 INFO:teuthology.orchestra.run.smithi087.stdout: "master_zone": "4b1d6266-9f83-43ad-a2df-8910ecae0ec1", 2024-04-02T15:31:16.170 INFO:teuthology.orchestra.run.smithi087.stdout: "zones": [ 2024-04-02T15:31:16.170 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-02T15:31:16.170 INFO:teuthology.orchestra.run.smithi087.stdout: "id": "4b1d6266-9f83-43ad-a2df-8910ecae0ec1", 2024-04-02T15:31:16.170 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "z", 2024-04-02T15:31:16.170 INFO:teuthology.orchestra.run.smithi087.stdout: "endpoints": [], 2024-04-02T15:31:16.170 INFO:teuthology.orchestra.run.smithi087.stdout: "log_meta": "false", 2024-04-02T15:31:16.170 INFO:teuthology.orchestra.run.smithi087.stdout: "log_data": "false", 2024-04-02T15:31:16.170 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_index_max_shards": 11, 2024-04-02T15:31:16.170 INFO:teuthology.orchestra.run.smithi087.stdout: "read_only": "false", 2024-04-02T15:31:16.170 INFO:teuthology.orchestra.run.smithi087.stdout: "tier_type": "", 2024-04-02T15:31:16.170 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from_all": "true", 2024-04-02T15:31:16.171 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_from": [], 2024-04-02T15:31:16.171 INFO:teuthology.orchestra.run.smithi087.stdout: "redirect_zone": "" 2024-04-02T15:31:16.171 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-02T15:31:16.171 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2024-04-02T15:31:16.171 INFO:teuthology.orchestra.run.smithi087.stdout: "placement_targets": [ 2024-04-02T15:31:16.171 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-02T15:31:16.171 INFO:teuthology.orchestra.run.smithi087.stdout: "name": "default-placement", 2024-04-02T15:31:16.171 INFO:teuthology.orchestra.run.smithi087.stdout: "tags": [], 2024-04-02T15:31:16.171 INFO:teuthology.orchestra.run.smithi087.stdout: "storage_classes": [ 2024-04-02T15:31:16.171 INFO:teuthology.orchestra.run.smithi087.stdout: "STANDARD" 2024-04-02T15:31:16.171 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-02T15:31:16.171 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-02T15:31:16.171 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2024-04-02T15:31:16.171 INFO:teuthology.orchestra.run.smithi087.stdout: "default_placement": "default-placement", 2024-04-02T15:31:16.171 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_id": "9bfe5281-8e8b-4ce7-adf3-af6510edad27", 2024-04-02T15:31:16.171 INFO:teuthology.orchestra.run.smithi087.stdout: "sync_policy": { 2024-04-02T15:31:16.172 INFO:teuthology.orchestra.run.smithi087.stdout: "groups": [] 2024-04-02T15:31:16.172 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-02T15:31:16.172 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-02T15:31:16.172 INFO:teuthology.orchestra.run.smithi087.stdout: ], 2024-04-02T15:31:16.172 INFO:teuthology.orchestra.run.smithi087.stdout: "short_zone_ids": [ 2024-04-02T15:31:16.172 INFO:teuthology.orchestra.run.smithi087.stdout: { 2024-04-02T15:31:16.172 INFO:teuthology.orchestra.run.smithi087.stdout: "key": "4b1d6266-9f83-43ad-a2df-8910ecae0ec1", 2024-04-02T15:31:16.172 INFO:teuthology.orchestra.run.smithi087.stdout: "val": 1247557524 2024-04-02T15:31:16.172 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-02T15:31:16.172 INFO:teuthology.orchestra.run.smithi087.stdout: ] 2024-04-02T15:31:16.172 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-02T15:31:16.172 INFO:teuthology.orchestra.run.smithi087.stdout: "master_zonegroup": "f3e98b35-7f05-4a1a-a5b0-45b7ebc272af", 2024-04-02T15:31:16.172 INFO:teuthology.orchestra.run.smithi087.stdout: "master_zone": "4b1d6266-9f83-43ad-a2df-8910ecae0ec1", 2024-04-02T15:31:16.172 INFO:teuthology.orchestra.run.smithi087.stdout: "period_config": { 2024-04-02T15:31:16.172 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_quota": { 2024-04-02T15:31:16.172 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false, 2024-04-02T15:31:16.173 INFO:teuthology.orchestra.run.smithi087.stdout: "check_on_raw": false, 2024-04-02T15:31:16.173 INFO:teuthology.orchestra.run.smithi087.stdout: "max_size": -1, 2024-04-02T15:31:16.173 INFO:teuthology.orchestra.run.smithi087.stdout: "max_size_kb": 0, 2024-04-02T15:31:16.173 INFO:teuthology.orchestra.run.smithi087.stdout: "max_objects": -1 2024-04-02T15:31:16.173 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-02T15:31:16.173 INFO:teuthology.orchestra.run.smithi087.stdout: "user_quota": { 2024-04-02T15:31:16.173 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false, 2024-04-02T15:31:16.173 INFO:teuthology.orchestra.run.smithi087.stdout: "check_on_raw": false, 2024-04-02T15:31:16.173 INFO:teuthology.orchestra.run.smithi087.stdout: "max_size": -1, 2024-04-02T15:31:16.173 INFO:teuthology.orchestra.run.smithi087.stdout: "max_size_kb": 0, 2024-04-02T15:31:16.173 INFO:teuthology.orchestra.run.smithi087.stdout: "max_objects": -1 2024-04-02T15:31:16.173 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-02T15:31:16.173 INFO:teuthology.orchestra.run.smithi087.stdout: "user_ratelimit": { 2024-04-02T15:31:16.173 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_ops": 0, 2024-04-02T15:31:16.173 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_ops": 0, 2024-04-02T15:31:16.174 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_bytes": 0, 2024-04-02T15:31:16.174 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_bytes": 0, 2024-04-02T15:31:16.174 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false 2024-04-02T15:31:16.174 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-02T15:31:16.174 INFO:teuthology.orchestra.run.smithi087.stdout: "bucket_ratelimit": { 2024-04-02T15:31:16.174 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_ops": 0, 2024-04-02T15:31:16.174 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_ops": 0, 2024-04-02T15:31:16.174 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_bytes": 0, 2024-04-02T15:31:16.174 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_bytes": 0, 2024-04-02T15:31:16.174 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false 2024-04-02T15:31:16.174 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-02T15:31:16.174 INFO:teuthology.orchestra.run.smithi087.stdout: "anonymous_ratelimit": { 2024-04-02T15:31:16.174 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_ops": 0, 2024-04-02T15:31:16.174 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_ops": 0, 2024-04-02T15:31:16.174 INFO:teuthology.orchestra.run.smithi087.stdout: "max_read_bytes": 0, 2024-04-02T15:31:16.174 INFO:teuthology.orchestra.run.smithi087.stdout: "max_write_bytes": 0, 2024-04-02T15:31:16.175 INFO:teuthology.orchestra.run.smithi087.stdout: "enabled": false 2024-04-02T15:31:16.175 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-02T15:31:16.175 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-02T15:31:16.175 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_id": "9bfe5281-8e8b-4ce7-adf3-af6510edad27", 2024-04-02T15:31:16.175 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_name": "r", 2024-04-02T15:31:16.175 INFO:teuthology.orchestra.run.smithi087.stdout: "realm_epoch": 2 2024-04-02T15:31:16.175 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-02T15:31:16.266 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:16 smithi087 ceph-mon[29707]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-04-02T15:31:16.266 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:16 smithi087 ceph-mon[29707]: osdmap e69: 8 total, 8 up, 8 in 2024-04-02T15:31:16.266 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:16 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/3391646244' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-02T15:31:16.267 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:16 smithi087 ceph-mon[29707]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-02T15:31:16.267 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:16 smithi087 ceph-mon[24900]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-04-02T15:31:16.267 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:16 smithi087 ceph-mon[24900]: osdmap e69: 8 total, 8 up, 8 in 2024-04-02T15:31:16.267 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:16 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/3391646244' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-02T15:31:16.267 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:16 smithi087 ceph-mon[24900]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-02T15:31:16.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:16 smithi106 ceph-mon[25843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-04-02T15:31:16.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:16 smithi106 ceph-mon[25843]: osdmap e69: 8 total, 8 up, 8 in 2024-04-02T15:31:16.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:16 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/3391646244' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-02T15:31:16.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:16 smithi106 ceph-mon[25843]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-02T15:31:16.620 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:31:16 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:31:16] "GET /metrics HTTP/1.1" 200 192769 "" "Prometheus/2.33.4" 2024-04-02T15:31:16.923 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000' 2024-04-02T15:31:17.212 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:17 smithi087 ceph-mon[29707]: pgmap v63: 129 pgs: 32 unknown, 32 creating+peering, 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:31:17.212 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:17 smithi087 ceph-mon[29707]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-04-02T15:31:17.212 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:17 smithi087 ceph-mon[29707]: osdmap e70: 8 total, 8 up, 8 in 2024-04-02T15:31:17.212 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:17 smithi087 ceph-mon[24900]: pgmap v63: 129 pgs: 32 unknown, 32 creating+peering, 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:31:17.213 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:17 smithi087 ceph-mon[24900]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-04-02T15:31:17.213 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:17 smithi087 ceph-mon[24900]: osdmap e70: 8 total, 8 up, 8 in 2024-04-02T15:31:17.366 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:17 smithi106 systemd[1]: Stopping Ceph prometheus.a for 0400842c-f105-11ee-b647-cb9ed24678a4... 2024-04-02T15:31:17.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:17 smithi106 ceph-mon[25843]: pgmap v63: 129 pgs: 32 unknown, 32 creating+peering, 65 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:31:17.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:17 smithi106 ceph-mon[25843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-04-02T15:31:17.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:17 smithi106 ceph-mon[25843]: osdmap e70: 8 total, 8 up, 8 in 2024-04-02T15:31:17.866 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:17 smithi106 bash[45344]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus.a" found: no such container 2024-04-02T15:31:17.867 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:17 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[43401]: ts=2024-04-02T15:31:17.614Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-02T15:31:17.867 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:17 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[43401]: ts=2024-04-02T15:31:17.614Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-02T15:31:17.867 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:17 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[43401]: ts=2024-04-02T15:31:17.614Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-02T15:31:17.867 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:17 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[43401]: ts=2024-04-02T15:31:17.614Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-02T15:31:17.867 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:17 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[43401]: ts=2024-04-02T15:31:17.614Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-02T15:31:17.867 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:17 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[43401]: ts=2024-04-02T15:31:17.614Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-02T15:31:17.867 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:17 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[43401]: ts=2024-04-02T15:31:17.614Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-02T15:31:17.867 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:17 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[43401]: ts=2024-04-02T15:31:17.614Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-02T15:31:17.867 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:17 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[43401]: ts=2024-04-02T15:31:17.614Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-02T15:31:17.867 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:17 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[43401]: ts=2024-04-02T15:31:17.615Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-02T15:31:17.867 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:17 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[43401]: ts=2024-04-02T15:31:17.615Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-02T15:31:17.867 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:17 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[43401]: ts=2024-04-02T15:31:17.615Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-02T15:31:17.868 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:17 smithi106 podman[45385]: 2024-04-02 15:31:17.624449104 +0000 UTC m=+0.093256273 container died 413397b51ac34273e87ef1c91905012d78db7c80a6e171ae552a8e219e59f967 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T15:31:18.156 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:17 smithi106 podman[45385]: 2024-04-02 15:31:17.955386119 +0000 UTC m=+0.424193299 container cleanup 413397b51ac34273e87ef1c91905012d78db7c80a6e171ae552a8e219e59f967 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T15:31:18.157 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:17 smithi106 bash[45385]: ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a 2024-04-02T15:31:18.157 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:18 smithi106 podman[45413]: 2024-04-02 15:31:18.139964047 +0000 UTC m=+0.518206458 container remove 413397b51ac34273e87ef1c91905012d78db7c80a6e171ae552a8e219e59f967 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T15:31:18.493 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:18 smithi106 bash[45425]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus.a" found: no such container 2024-04-02T15:31:18.493 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:18 smithi106 systemd[1]: ceph-0400842c-f105-11ee-b647-cb9ed24678a4@prometheus.a.service: Deactivated successfully. 2024-04-02T15:31:18.493 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:18 smithi106 systemd[1]: Stopped Ceph prometheus.a for 0400842c-f105-11ee-b647-cb9ed24678a4. 2024-04-02T15:31:18.493 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:18 smithi106 systemd[1]: Starting Ceph prometheus.a for 0400842c-f105-11ee-b647-cb9ed24678a4... 2024-04-02T15:31:18.897 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:31:18 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:31:18] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:31:19.108 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:19 smithi106 podman[45550]: 2024-04-02 15:31:18.927686879 +0000 UTC m=+0.018416305 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-02T15:31:19.108 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:19 smithi106 podman[45550]: 2024-04-02 15:31:19.028101104 +0000 UTC m=+0.118830522 container create 3ea0ba4743fb84bbe082ed969e1ff6c636d24c050b93715af45c7dddcd0b5834 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T15:31:19.322 INFO:teuthology.orchestra.run.smithi087.stdout:Scheduled rgw.foo update... 2024-04-02T15:31:19.323 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:19 smithi087 ceph-mon[29707]: pgmap v65: 129 pgs: 129 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 2.2 KiB/s wr, 5 op/s 2024-04-02T15:31:19.323 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:19 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:19.323 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:19 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:19.323 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:19 smithi087 ceph-mon[24900]: pgmap v65: 129 pgs: 129 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 2.2 KiB/s wr, 5 op/s 2024-04-02T15:31:19.323 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:19 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:19.323 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:19 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:19.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:19 smithi106 ceph-mon[25843]: pgmap v65: 129 pgs: 129 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 2.2 KiB/s wr, 5 op/s 2024-04-02T15:31:19.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:19 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:19.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:19 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:19.367 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:19 smithi106 podman[45550]: 2024-04-02 15:31:19.213709243 +0000 UTC m=+0.304438660 container init 3ea0ba4743fb84bbe082ed969e1ff6c636d24c050b93715af45c7dddcd0b5834 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T15:31:19.367 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:19 smithi106 podman[45550]: 2024-04-02 15:31:19.216137073 +0000 UTC m=+0.306866490 container start 3ea0ba4743fb84bbe082ed969e1ff6c636d24c050b93715af45c7dddcd0b5834 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T15:31:19.367 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:19 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[45564]: ts=2024-04-02T15:31:19.237Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-02T15:31:19.367 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:19 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[45564]: ts=2024-04-02T15:31:19.238Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-02T15:31:19.367 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:19 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[45564]: ts=2024-04-02T15:31:19.238Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-02T15:31:19.367 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:19 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[45564]: ts=2024-04-02T15:31:19.238Z caller=main.go:518 level=info host_details="(Linux 5.14.0-434.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Mar 26 19:20:47 UTC 2024 x86_64 smithi106 (none))" 2024-04-02T15:31:19.367 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:19 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[45564]: ts=2024-04-02T15:31:19.238Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-02T15:31:19.368 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:19 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[45564]: ts=2024-04-02T15:31:19.238Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-02T15:31:19.368 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:19 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[45564]: ts=2024-04-02T15:31:19.239Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-02T15:31:19.368 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:19 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[45564]: ts=2024-04-02T15:31:19.240Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-02T15:31:19.368 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:19 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[45564]: ts=2024-04-02T15:31:19.240Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-02T15:31:19.368 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:19 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[45564]: ts=2024-04-02T15:31:19.242Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-02T15:31:19.368 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:19 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[45564]: ts=2024-04-02T15:31:19.242Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.44µs 2024-04-02T15:31:19.368 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:19 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[45564]: ts=2024-04-02T15:31:19.242Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-02T15:31:19.368 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:19 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[45564]: ts=2024-04-02T15:31:19.242Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-04-02T15:31:19.368 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:19 smithi106 bash[45550]: 3ea0ba4743fb84bbe082ed969e1ff6c636d24c050b93715af45c7dddcd0b5834 2024-04-02T15:31:19.368 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:19 smithi106 systemd[1]: Started Ceph prometheus.a for 0400842c-f105-11ee-b647-cb9ed24678a4. 2024-04-02T15:31:19.825 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph osd pool create foo' 2024-04-02T15:31:20.116 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:19 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[45564]: ts=2024-04-02T15:31:19.629Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-04-02T15:31:20.116 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:19 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[45564]: ts=2024-04-02T15:31:19.629Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-04-02T15:31:20.116 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:19 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[45564]: ts=2024-04-02T15:31:19.629Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=27.582µs wal_replay_duration=387.566503ms total_replay_duration=387.607093ms 2024-04-02T15:31:20.117 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:19 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[45564]: ts=2024-04-02T15:31:19.632Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-02T15:31:20.117 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:19 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[45564]: ts=2024-04-02T15:31:19.632Z caller=main.go:947 level=info msg="TSDB started" 2024-04-02T15:31:20.117 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:19 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[45564]: ts=2024-04-02T15:31:19.632Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-02T15:31:20.117 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:19 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[45564]: ts=2024-04-02T15:31:19.648Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.70712ms db_storage=701ns remote_storage=1.156µs web_handler=344ns query_engine=757ns scrape=391.899µs scrape_sd=24.455µs notify=26.526µs notify_sd=9.481µs rules=15.022364ms 2024-04-02T15:31:20.117 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:31:19 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[45564]: ts=2024-04-02T15:31:19.648Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-02T15:31:20.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:20 smithi106 ceph-mon[25843]: pgmap v66: 129 pgs: 129 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.5 KiB/s wr, 3 op/s 2024-04-02T15:31:20.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:20 smithi106 ceph-mon[25843]: from='client.24592 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:31:20.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:20 smithi106 ceph-mon[25843]: Saving service rgw.foo spec with placement count:2 2024-04-02T15:31:20.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:20 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:20.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:20 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:20.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:20 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-02T15:31:20.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:20 smithi106 ceph-mon[25843]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-02T15:31:20.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:20 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.87:9093"}]: dispatch 2024-04-02T15:31:20.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:20 smithi106 ceph-mon[25843]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.87:9093"}]: dispatch 2024-04-02T15:31:20.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:20 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:20.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:20 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T15:31:20.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:20 smithi106 ceph-mon[25843]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T15:31:20.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:20 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-02T15:31:20.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:20 smithi106 ceph-mon[25843]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-02T15:31:20.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:20 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.106:3000"}]: dispatch 2024-04-02T15:31:20.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:20 smithi106 ceph-mon[25843]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.106:3000"}]: dispatch 2024-04-02T15:31:20.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:20 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:20.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:20 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:31:20.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:20 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:31:20.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:20 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:31:20.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:20 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:20.619 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:20 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:20.619 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:20 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:20.619 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:20 smithi106 ceph-mon[25843]: Saving service rgw.foo spec with placement count:2 2024-04-02T15:31:20.619 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:20 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:20.619 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:20 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi087.awfxzs", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T15:31:20.619 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:20 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi087.awfxzs", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-02T15:31:20.619 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:20 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:20.619 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:20 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:31:20.619 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:20 smithi106 ceph-mon[25843]: Deploying daemon rgw.foo.smithi087.awfxzs on smithi087 2024-04-02T15:31:20.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[29707]: pgmap v66: 129 pgs: 129 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.5 KiB/s wr, 3 op/s 2024-04-02T15:31:20.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[29707]: from='client.24592 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:31:20.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[29707]: Saving service rgw.foo spec with placement count:2 2024-04-02T15:31:20.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:20.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:20.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-02T15:31:20.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[29707]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-02T15:31:20.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.87:9093"}]: dispatch 2024-04-02T15:31:20.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[29707]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.87:9093"}]: dispatch 2024-04-02T15:31:20.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:20.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T15:31:20.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[29707]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T15:31:20.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-02T15:31:20.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[29707]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-02T15:31:20.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.106:3000"}]: dispatch 2024-04-02T15:31:20.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[29707]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.106:3000"}]: dispatch 2024-04-02T15:31:20.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:20.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:31:20.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:31:20.623 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:31:20.623 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:20.623 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:20.623 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:20.623 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[29707]: Saving service rgw.foo spec with placement count:2 2024-04-02T15:31:20.623 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:20.623 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi087.awfxzs", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T15:31:20.623 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi087.awfxzs", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-02T15:31:20.623 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[24900]: pgmap v66: 129 pgs: 129 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.5 KiB/s wr, 3 op/s 2024-04-02T15:31:20.624 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[24900]: from='client.24592 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:31:20.624 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[24900]: Saving service rgw.foo spec with placement count:2 2024-04-02T15:31:20.624 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:20.624 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:20.624 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-02T15:31:20.624 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[24900]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-02T15:31:20.624 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.87:9093"}]: dispatch 2024-04-02T15:31:20.624 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[24900]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.87:9093"}]: dispatch 2024-04-02T15:31:20.624 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:20.625 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T15:31:20.625 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[24900]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T15:31:20.625 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-02T15:31:20.625 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[24900]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-02T15:31:20.625 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.106:3000"}]: dispatch 2024-04-02T15:31:20.625 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[24900]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.106:3000"}]: dispatch 2024-04-02T15:31:20.625 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:20.625 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:31:20.625 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:31:20.625 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:31:20.625 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:20.626 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:20.626 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:20.626 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[24900]: Saving service rgw.foo spec with placement count:2 2024-04-02T15:31:20.626 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:20.626 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi087.awfxzs", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T15:31:20.626 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi087.awfxzs", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-02T15:31:20.627 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:20.627 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:31:20.627 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[29707]: Deploying daemon rgw.foo.smithi087.awfxzs on smithi087 2024-04-02T15:31:20.627 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:20.627 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:31:20.627 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:20 smithi087 ceph-mon[24900]: Deploying daemon rgw.foo.smithi087.awfxzs on smithi087 2024-04-02T15:31:22.525 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:22 smithi087 ceph-mon[29707]: pgmap v67: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-04-02T15:31:22.526 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:22 smithi087 ceph-mon[24900]: pgmap v67: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-04-02T15:31:22.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:22 smithi106 ceph-mon[25843]: pgmap v67: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-04-02T15:31:23.094 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:31:23 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=info ts=2024-04-02T15:31:23.015Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002277628s 2024-04-02T15:31:23.354 INFO:teuthology.orchestra.run.smithi087.stderr:pool 'foo' created 2024-04-02T15:31:23.354 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:23 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/3437256430' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-02T15:31:23.354 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:23 smithi087 ceph-mon[24900]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-02T15:31:23.355 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:23 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/3437256430' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-02T15:31:23.355 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:23 smithi087 ceph-mon[29707]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-02T15:31:23.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:23 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/3437256430' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-02T15:31:23.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:23 smithi106 ceph-mon[25843]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-02T15:31:24.005 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'rbd pool init foo' 2024-04-02T15:31:24.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:24 smithi106 ceph-mon[25843]: pgmap v68: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1006 B/s rd, 1.1 KiB/s wr, 2 op/s 2024-04-02T15:31:24.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:24 smithi106 ceph-mon[25843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-04-02T15:31:24.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:24 smithi106 ceph-mon[25843]: osdmap e71: 8 total, 8 up, 8 in 2024-04-02T15:31:24.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:24 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:24.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:24 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi106.bfeikp", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T15:31:24.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:24 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi106.bfeikp", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-02T15:31:24.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:24 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:24.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:24 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:31:24.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:24 smithi106 ceph-mon[25843]: Deploying daemon rgw.foo.smithi106.bfeikp on smithi106 2024-04-02T15:31:24.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:24 smithi087 ceph-mon[29707]: pgmap v68: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1006 B/s rd, 1.1 KiB/s wr, 2 op/s 2024-04-02T15:31:24.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:24 smithi087 ceph-mon[29707]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-04-02T15:31:24.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:24 smithi087 ceph-mon[29707]: osdmap e71: 8 total, 8 up, 8 in 2024-04-02T15:31:24.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:24 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:24.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:24 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi106.bfeikp", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T15:31:24.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:24 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi106.bfeikp", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-02T15:31:24.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:24 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:24.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:24 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:31:24.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:24 smithi087 ceph-mon[29707]: Deploying daemon rgw.foo.smithi106.bfeikp on smithi106 2024-04-02T15:31:24.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:24 smithi087 ceph-mon[24900]: pgmap v68: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1006 B/s rd, 1.1 KiB/s wr, 2 op/s 2024-04-02T15:31:24.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:24 smithi087 ceph-mon[24900]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-04-02T15:31:24.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:24 smithi087 ceph-mon[24900]: osdmap e71: 8 total, 8 up, 8 in 2024-04-02T15:31:24.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:24 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:24.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:24 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi106.bfeikp", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-02T15:31:24.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:24 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi106.bfeikp", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-02T15:31:24.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:24 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:24.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:24 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:31:24.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:24 smithi087 ceph-mon[24900]: Deploying daemon rgw.foo.smithi106.bfeikp on smithi106 2024-04-02T15:31:25.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:25 smithi106 ceph-mon[25843]: osdmap e72: 8 total, 8 up, 8 in 2024-04-02T15:31:25.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:25 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:25.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:25 smithi087 ceph-mon[29707]: osdmap e72: 8 total, 8 up, 8 in 2024-04-02T15:31:25.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:25 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:25.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:25 smithi087 ceph-mon[24900]: osdmap e72: 8 total, 8 up, 8 in 2024-04-02T15:31:25.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:25 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:26.355 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:31:26 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:31:26] "GET /metrics HTTP/1.1" 200 196815 "" "Prometheus/2.33.4" 2024-04-02T15:31:26.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:26 smithi106 ceph-mon[25843]: pgmap v71: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:31:26.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:26 smithi106 ceph-mon[25843]: osdmap e73: 8 total, 8 up, 8 in 2024-04-02T15:31:26.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:26 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/1586744059' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-02T15:31:26.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:26 smithi106 ceph-mon[25843]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-02T15:31:26.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:26 smithi087 ceph-mon[29707]: pgmap v71: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:31:26.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:26 smithi087 ceph-mon[29707]: osdmap e73: 8 total, 8 up, 8 in 2024-04-02T15:31:26.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:26 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/1586744059' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-02T15:31:26.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:26 smithi087 ceph-mon[29707]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-02T15:31:26.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:26 smithi087 ceph-mon[24900]: pgmap v71: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:31:26.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:26 smithi087 ceph-mon[24900]: osdmap e73: 8 total, 8 up, 8 in 2024-04-02T15:31:26.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:26 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/1586744059' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-02T15:31:26.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:26 smithi087 ceph-mon[24900]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-02T15:31:27.493 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:27 smithi106 ceph-mon[25843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-04-02T15:31:27.493 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:27 smithi106 ceph-mon[25843]: osdmap e74: 8 total, 8 up, 8 in 2024-04-02T15:31:27.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:27 smithi087 ceph-mon[29707]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-04-02T15:31:27.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:27 smithi087 ceph-mon[29707]: osdmap e74: 8 total, 8 up, 8 in 2024-04-02T15:31:27.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:27 smithi087 ceph-mon[24900]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-04-02T15:31:27.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:27 smithi087 ceph-mon[24900]: osdmap e74: 8 total, 8 up, 8 in 2024-04-02T15:31:28.101 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:31:27 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:31:27] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:31:28.367 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:31:28 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:31:28] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:31:28.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:28 smithi106 ceph-mon[25843]: pgmap v74: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 116 KiB/s rd, 8.2 KiB/s wr, 280 op/s 2024-04-02T15:31:28.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:28 smithi106 ceph-mon[25843]: osdmap e75: 8 total, 8 up, 8 in 2024-04-02T15:31:28.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:28 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:28.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:28 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:28.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:28 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:28.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:28 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:28.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:28 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:28.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:28 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:31:28.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:28 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:31:28.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:28 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:31:28.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:28 smithi087 ceph-mon[24900]: pgmap v74: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 116 KiB/s rd, 8.2 KiB/s wr, 280 op/s 2024-04-02T15:31:28.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:28 smithi087 ceph-mon[24900]: osdmap e75: 8 total, 8 up, 8 in 2024-04-02T15:31:28.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:28 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:28.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:28 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:28.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:28 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:28.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:28 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:28.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:28 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:28.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:28 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:31:28.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:28 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:31:28.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:28 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:31:28.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:28 smithi087 ceph-mon[29707]: pgmap v74: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 116 KiB/s rd, 8.2 KiB/s wr, 280 op/s 2024-04-02T15:31:28.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:28 smithi087 ceph-mon[29707]: osdmap e75: 8 total, 8 up, 8 in 2024-04-02T15:31:28.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:28 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:28.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:28 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:28.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:28 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:28.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:28 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:28.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:28 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:28.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:28 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:31:28.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:28 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:31:28.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:28 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:31:29.189 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph orch apply iscsi foo u p' 2024-04-02T15:31:29.468 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:29 smithi106 ceph-mon[25843]: Checking dashboard <-> RGW credentials 2024-04-02T15:31:29.468 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:29 smithi106 ceph-mon[25843]: osdmap e76: 8 total, 8 up, 8 in 2024-04-02T15:31:29.468 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:29 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:29.468 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:29 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:29.468 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:29 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:29.468 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:29 smithi106 ceph-mon[25843]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-02T15:31:29.468 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:29 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:31:29.468 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:29 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:31:29.469 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:29 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:31:29.469 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:29 smithi106 ceph-mon[25843]: Checking dashboard <-> RGW credentials 2024-04-02T15:31:29.469 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:29 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:29.469 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:29 smithi106 ceph-mon[25843]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-02T15:31:29.510 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:29 smithi087 ceph-mon[24900]: Checking dashboard <-> RGW credentials 2024-04-02T15:31:29.511 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:29 smithi087 ceph-mon[24900]: osdmap e76: 8 total, 8 up, 8 in 2024-04-02T15:31:29.511 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:29 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:29.511 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:29 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:29.511 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:29 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:29.511 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:29 smithi087 ceph-mon[24900]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-02T15:31:29.511 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:29 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:31:29.511 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:29 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:31:29.511 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:29 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:31:29.511 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:29 smithi087 ceph-mon[24900]: Checking dashboard <-> RGW credentials 2024-04-02T15:31:29.511 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:29 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:29.511 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:29 smithi087 ceph-mon[24900]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-02T15:31:29.512 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:29 smithi087 ceph-mon[29707]: Checking dashboard <-> RGW credentials 2024-04-02T15:31:29.512 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:29 smithi087 ceph-mon[29707]: osdmap e76: 8 total, 8 up, 8 in 2024-04-02T15:31:29.512 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:29 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:29.512 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:29 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:29.512 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:29 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:29.512 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:29 smithi087 ceph-mon[29707]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-02T15:31:29.512 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:29 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:31:29.512 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:29 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:31:29.512 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:29 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:31:29.512 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:29 smithi087 ceph-mon[29707]: Checking dashboard <-> RGW credentials 2024-04-02T15:31:29.512 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:29 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:29.512 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:29 smithi087 ceph-mon[29707]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-02T15:31:30.315 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:31:30 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:31:30] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:31:30.481 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:30 smithi106 ceph-mon[25843]: pgmap v77: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 116 KiB/s rd, 8.2 KiB/s wr, 280 op/s 2024-04-02T15:31:30.481 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:30 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:30.481 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:30 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:30.481 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:30 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:30.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:30 smithi087 ceph-mon[29707]: pgmap v77: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 116 KiB/s rd, 8.2 KiB/s wr, 280 op/s 2024-04-02T15:31:30.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:30 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:30.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:30 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:30.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:30 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:30.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:30 smithi087 ceph-mon[24900]: pgmap v77: 161 pgs: 161 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 116 KiB/s rd, 8.2 KiB/s wr, 280 op/s 2024-04-02T15:31:30.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:30 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:30.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:30 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:30.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:30 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:32.502 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:32 smithi087 ceph-mon[29707]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 202 KiB/s rd, 6.5 KiB/s wr, 374 op/s 2024-04-02T15:31:32.503 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:32 smithi087 ceph-mon[24900]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 202 KiB/s rd, 6.5 KiB/s wr, 374 op/s 2024-04-02T15:31:32.599 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:32 smithi106 ceph-mon[25843]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 202 KiB/s rd, 6.5 KiB/s wr, 374 op/s 2024-04-02T15:31:32.811 INFO:teuthology.orchestra.run.smithi087.stdout:Scheduled iscsi.foo update... 2024-04-02T15:31:33.370 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:31:33 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:31:33] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:31:33.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:33 smithi087 ceph-mon[24900]: from='client.24724 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:31:33.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:33 smithi087 ceph-mon[24900]: Saving service iscsi.foo spec with placement count:1 2024-04-02T15:31:33.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:33 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:33.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:33 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:31:33.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:33 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:31:33.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:33 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:31:33.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:33 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:33.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:33 smithi087 ceph-mon[24900]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-02T15:31:33.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:33 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:33.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:33 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:33.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:33 smithi087 ceph-mon[29707]: from='client.24724 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:31:33.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:33 smithi087 ceph-mon[29707]: Saving service iscsi.foo spec with placement count:1 2024-04-02T15:31:33.872 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:33 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:33.872 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:33 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:31:33.872 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:33 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:31:33.872 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:33 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:31:33.872 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:33 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:33.872 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:33 smithi087 ceph-mon[29707]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-02T15:31:33.872 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:33 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:33.872 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:33 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:33.987 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'sleep 180' 2024-04-02T15:31:34.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:33 smithi106 ceph-mon[25843]: from='client.24724 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:31:34.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:33 smithi106 ceph-mon[25843]: Saving service iscsi.foo spec with placement count:1 2024-04-02T15:31:34.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:33 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:34.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:33 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:31:34.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:33 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:31:34.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:33 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:31:34.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:33 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:34.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:33 smithi106 ceph-mon[25843]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-02T15:31:34.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:33 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:34.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:33 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:34.117 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:31:33 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:31:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:31:34.810 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:31:34 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:31:34] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:31:35.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:34 smithi106 ceph-mon[25843]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 105 KiB/s rd, 744 B/s wr, 157 op/s 2024-04-02T15:31:35.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:34 smithi106 ceph-mon[25843]: osdmap e77: 8 total, 8 up, 8 in 2024-04-02T15:31:35.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:34 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:35.122 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:34 smithi087 ceph-mon[24900]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 105 KiB/s rd, 744 B/s wr, 157 op/s 2024-04-02T15:31:35.122 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:34 smithi087 ceph-mon[24900]: osdmap e77: 8 total, 8 up, 8 in 2024-04-02T15:31:35.122 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:34 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:35.122 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:34 smithi087 ceph-mon[29707]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 105 KiB/s rd, 744 B/s wr, 157 op/s 2024-04-02T15:31:35.122 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:34 smithi087 ceph-mon[29707]: osdmap e77: 8 total, 8 up, 8 in 2024-04-02T15:31:35.122 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:34 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:36.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:35 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.4", "id": [7, 2]}]: dispatch 2024-04-02T15:31:36.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:35 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-04-02T15:31:36.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:35 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-04-02T15:31:36.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:35 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-04-02T15:31:36.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:35 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:31:36.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:35 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:31:36.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:35 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:36.120 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:31:35 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:31:35] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:31:36.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:35 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.4", "id": [7, 2]}]: dispatch 2024-04-02T15:31:36.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:35 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-04-02T15:31:36.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:35 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-04-02T15:31:36.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:35 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-04-02T15:31:36.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:35 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:31:36.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:35 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:31:36.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:35 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:36.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:35 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.4", "id": [7, 2]}]: dispatch 2024-04-02T15:31:36.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:35 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-04-02T15:31:36.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:35 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-04-02T15:31:36.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:35 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-04-02T15:31:36.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:35 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:31:36.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:35 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:31:36.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:35 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:36.579 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:31:36 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:31:36] "GET /metrics HTTP/1.1" 200 207981 "" "Prometheus/2.33.4" 2024-04-02T15:31:37.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:36 smithi106 ceph-mon[25843]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 92 KiB/s rd, 650 B/s wr, 137 op/s 2024-04-02T15:31:37.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.4", "id": [7, 2]}]': finished 2024-04-02T15:31:37.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-04-02T15:31:37.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2024-04-02T15:31:37.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-04-02T15:31:37.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:36 smithi106 ceph-mon[25843]: osdmap e78: 8 total, 8 up, 8 in 2024-04-02T15:31:37.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:37.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:37.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:31:37.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:31:37.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:31:37.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:37.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:36 smithi106 ceph-mon[25843]: Checking pool "foo" exists for service iscsi.foo 2024-04-02T15:31:37.118 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi087.icnqbx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T15:31:37.118 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi087.icnqbx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-04-02T15:31:37.118 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:36 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:31:37.118 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:36 smithi106 ceph-mon[25843]: Deploying daemon iscsi.foo.smithi087.icnqbx on smithi087 2024-04-02T15:31:37.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:36 smithi087 ceph-mon[24900]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 92 KiB/s rd, 650 B/s wr, 137 op/s 2024-04-02T15:31:37.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.4", "id": [7, 2]}]': finished 2024-04-02T15:31:37.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-04-02T15:31:37.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2024-04-02T15:31:37.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-04-02T15:31:37.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:36 smithi087 ceph-mon[24900]: osdmap e78: 8 total, 8 up, 8 in 2024-04-02T15:31:37.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:37.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:37.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:31:37.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:31:37.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:31:37.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:37.122 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:36 smithi087 ceph-mon[24900]: Checking pool "foo" exists for service iscsi.foo 2024-04-02T15:31:37.122 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi087.icnqbx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T15:31:37.122 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi087.icnqbx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-04-02T15:31:37.122 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:36 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:31:37.122 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:36 smithi087 ceph-mon[24900]: Deploying daemon iscsi.foo.smithi087.icnqbx on smithi087 2024-04-02T15:31:37.122 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:36 smithi087 ceph-mon[29707]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 92 KiB/s rd, 650 B/s wr, 137 op/s 2024-04-02T15:31:37.122 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.4", "id": [7, 2]}]': finished 2024-04-02T15:31:37.122 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-04-02T15:31:37.122 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2024-04-02T15:31:37.122 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-04-02T15:31:37.123 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:36 smithi087 ceph-mon[29707]: osdmap e78: 8 total, 8 up, 8 in 2024-04-02T15:31:37.123 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:37.123 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:37.123 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:31:37.123 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:31:37.123 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:31:37.123 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:37.123 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:36 smithi087 ceph-mon[29707]: Checking pool "foo" exists for service iscsi.foo 2024-04-02T15:31:37.123 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi087.icnqbx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-02T15:31:37.123 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi087.icnqbx", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-04-02T15:31:37.123 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:36 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:31:37.123 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:36 smithi087 ceph-mon[29707]: Deploying daemon iscsi.foo.smithi087.icnqbx on smithi087 2024-04-02T15:31:37.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:37 smithi087 ceph-mon[24900]: osdmap e79: 8 total, 8 up, 8 in 2024-04-02T15:31:37.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:37 smithi087 ceph-mon[29707]: osdmap e79: 8 total, 8 up, 8 in 2024-04-02T15:31:38.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:37 smithi106 ceph-mon[25843]: osdmap e79: 8 total, 8 up, 8 in 2024-04-02T15:31:39.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:38 smithi106 ceph-mon[25843]: pgmap v84: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:31:39.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:38 smithi106 ceph-mon[25843]: mgrmap e20: y(active, since 2m), standbys: x 2024-04-02T15:31:39.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:38 smithi087 ceph-mon[29707]: pgmap v84: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:31:39.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:38 smithi087 ceph-mon[29707]: mgrmap e20: y(active, since 2m), standbys: x 2024-04-02T15:31:39.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:38 smithi087 ceph-mon[24900]: pgmap v84: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:31:39.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:38 smithi087 ceph-mon[24900]: mgrmap e20: y(active, since 2m), standbys: x 2024-04-02T15:31:40.026 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:40.026 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:31:40.026 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:31:40.026 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:31:40.027 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:40.027 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[29707]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-02T15:31:40.027 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T15:31:40.027 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[29707]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T15:31:40.027 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[29707]: Adding iSCSI gateway http://:@172.21.15.87:5000 to Dashboard 2024-04-02T15:31:40.027 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T15:31:40.027 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[29707]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T15:31:40.027 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:40.027 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi087"}]: dispatch 2024-04-02T15:31:40.027 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[29707]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi087"}]: dispatch 2024-04-02T15:31:40.027 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:40.027 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:31:40.027 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:31:40.027 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:31:40.028 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:40.028 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[29707]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-02T15:31:40.028 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/2317979896' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T15:31:40.028 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/119351802' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/3267436387"}]: dispatch 2024-04-02T15:31:40.028 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[29707]: from='client.? ' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/3267436387"}]: dispatch 2024-04-02T15:31:40.028 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:40.028 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:31:40.028 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:31:40.028 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:31:40.028 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:40.028 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[24900]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-02T15:31:40.028 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T15:31:40.028 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[24900]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T15:31:40.029 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[24900]: Adding iSCSI gateway http://:@172.21.15.87:5000 to Dashboard 2024-04-02T15:31:40.029 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T15:31:40.029 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[24900]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T15:31:40.029 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:40.029 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi087"}]: dispatch 2024-04-02T15:31:40.029 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[24900]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi087"}]: dispatch 2024-04-02T15:31:40.029 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:40.029 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:31:40.029 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:31:40.029 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:31:40.029 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:40.029 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[24900]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-02T15:31:40.029 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/2317979896' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T15:31:40.029 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/119351802' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/3267436387"}]: dispatch 2024-04-02T15:31:40.030 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:39 smithi087 ceph-mon[24900]: from='client.? ' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/3267436387"}]: dispatch 2024-04-02T15:31:40.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:39 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:40.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:39 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:31:40.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:39 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:31:40.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:39 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:31:40.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:39 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:40.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:39 smithi106 ceph-mon[25843]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-02T15:31:40.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:39 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T15:31:40.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:39 smithi106 ceph-mon[25843]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-02T15:31:40.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:39 smithi106 ceph-mon[25843]: Adding iSCSI gateway http://:@172.21.15.87:5000 to Dashboard 2024-04-02T15:31:40.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:39 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T15:31:40.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:39 smithi106 ceph-mon[25843]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-02T15:31:40.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:39 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:40.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:39 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi087"}]: dispatch 2024-04-02T15:31:40.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:39 smithi106 ceph-mon[25843]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi087"}]: dispatch 2024-04-02T15:31:40.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:39 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:40.368 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:39 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:31:40.368 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:39 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:31:40.368 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:39 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:31:40.368 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:39 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:40.368 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:39 smithi106 ceph-mon[25843]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-02T15:31:40.368 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:39 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/2317979896' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-02T15:31:40.368 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:39 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/119351802' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/3267436387"}]: dispatch 2024-04-02T15:31:40.368 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:39 smithi106 ceph-mon[25843]: from='client.? ' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/3267436387"}]: dispatch 2024-04-02T15:31:41.078 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:40 smithi087 ceph-mon[29707]: pgmap v85: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:31:41.078 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:40 smithi087 ceph-mon[29707]: from='client.? ' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/3267436387"}]': finished 2024-04-02T15:31:41.078 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:40 smithi087 ceph-mon[29707]: osdmap e80: 8 total, 8 up, 8 in 2024-04-02T15:31:41.078 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:40 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:41.078 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:40 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/72583274' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:6800/2108248912"}]: dispatch 2024-04-02T15:31:41.079 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:40 smithi087 ceph-mon[24900]: pgmap v85: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:31:41.079 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:40 smithi087 ceph-mon[24900]: from='client.? ' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/3267436387"}]': finished 2024-04-02T15:31:41.079 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:40 smithi087 ceph-mon[24900]: osdmap e80: 8 total, 8 up, 8 in 2024-04-02T15:31:41.079 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:40 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:41.079 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:40 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/72583274' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:6800/2108248912"}]: dispatch 2024-04-02T15:31:41.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:40 smithi106 ceph-mon[25843]: pgmap v85: 161 pgs: 4 peering, 157 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:31:41.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:40 smithi106 ceph-mon[25843]: from='client.? ' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/3267436387"}]': finished 2024-04-02T15:31:41.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:40 smithi106 ceph-mon[25843]: osdmap e80: 8 total, 8 up, 8 in 2024-04-02T15:31:41.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:40 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:41.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:40 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/72583274' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:6800/2108248912"}]: dispatch 2024-04-02T15:31:42.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:42 smithi106 ceph-mon[25843]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 170 B/s wr, 1 op/s; 67 B/s, 0 objects/s recovering 2024-04-02T15:31:42.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:42 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/72583274' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:6800/2108248912"}]': finished 2024-04-02T15:31:42.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:42 smithi106 ceph-mon[25843]: osdmap e81: 8 total, 8 up, 8 in 2024-04-02T15:31:42.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:42 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/3806114575' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/2189995990"}]: dispatch 2024-04-02T15:31:42.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:42 smithi087 ceph-mon[29707]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 170 B/s wr, 1 op/s; 67 B/s, 0 objects/s recovering 2024-04-02T15:31:42.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:42 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/72583274' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:6800/2108248912"}]': finished 2024-04-02T15:31:42.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:42 smithi087 ceph-mon[29707]: osdmap e81: 8 total, 8 up, 8 in 2024-04-02T15:31:42.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:42 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/3806114575' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/2189995990"}]: dispatch 2024-04-02T15:31:42.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:42 smithi087 ceph-mon[24900]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 170 B/s wr, 1 op/s; 67 B/s, 0 objects/s recovering 2024-04-02T15:31:42.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:42 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/72583274' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:6800/2108248912"}]': finished 2024-04-02T15:31:42.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:42 smithi087 ceph-mon[24900]: osdmap e81: 8 total, 8 up, 8 in 2024-04-02T15:31:42.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:42 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/3806114575' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/2189995990"}]: dispatch 2024-04-02T15:31:43.120 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:31:42 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:31:42] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:31:43.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:43 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/3806114575' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/2189995990"}]': finished 2024-04-02T15:31:43.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:43 smithi106 ceph-mon[25843]: osdmap e82: 8 total, 8 up, 8 in 2024-04-02T15:31:43.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:43 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/2789515592' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/1859893958"}]: dispatch 2024-04-02T15:31:43.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:43 smithi106 ceph-mon[25843]: from='client.? ' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/1859893958"}]: dispatch 2024-04-02T15:31:43.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:43 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:43.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:43 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/3806114575' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/2189995990"}]': finished 2024-04-02T15:31:43.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:43 smithi087 ceph-mon[24900]: osdmap e82: 8 total, 8 up, 8 in 2024-04-02T15:31:43.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:43 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/2789515592' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/1859893958"}]: dispatch 2024-04-02T15:31:43.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:43 smithi087 ceph-mon[24900]: from='client.? ' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/1859893958"}]: dispatch 2024-04-02T15:31:43.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:43 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:43.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:43 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/3806114575' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/2189995990"}]': finished 2024-04-02T15:31:43.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:43 smithi087 ceph-mon[29707]: osdmap e82: 8 total, 8 up, 8 in 2024-04-02T15:31:43.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:43 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/2789515592' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/1859893958"}]: dispatch 2024-04-02T15:31:43.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:43 smithi087 ceph-mon[29707]: from='client.? ' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/1859893958"}]: dispatch 2024-04-02T15:31:43.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:43 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:44.116 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:31:43 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:31:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:31:44.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:44 smithi087 ceph-mon[29707]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 170 B/s wr, 1 op/s; 67 B/s, 0 objects/s recovering 2024-04-02T15:31:44.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:44 smithi087 ceph-mon[29707]: from='client.? ' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/1859893958"}]': finished 2024-04-02T15:31:44.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:44 smithi087 ceph-mon[29707]: osdmap e83: 8 total, 8 up, 8 in 2024-04-02T15:31:44.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:44 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/3075918570' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:6800/3965460628"}]: dispatch 2024-04-02T15:31:44.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:44 smithi087 ceph-mon[24900]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 170 B/s wr, 1 op/s; 67 B/s, 0 objects/s recovering 2024-04-02T15:31:44.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:44 smithi087 ceph-mon[24900]: from='client.? ' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/1859893958"}]': finished 2024-04-02T15:31:44.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:44 smithi087 ceph-mon[24900]: osdmap e83: 8 total, 8 up, 8 in 2024-04-02T15:31:44.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:44 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/3075918570' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:6800/3965460628"}]: dispatch 2024-04-02T15:31:44.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:44 smithi106 ceph-mon[25843]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 170 B/s wr, 1 op/s; 67 B/s, 0 objects/s recovering 2024-04-02T15:31:44.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:44 smithi106 ceph-mon[25843]: from='client.? ' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/1859893958"}]': finished 2024-04-02T15:31:44.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:44 smithi106 ceph-mon[25843]: osdmap e83: 8 total, 8 up, 8 in 2024-04-02T15:31:44.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:44 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/3075918570' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:6800/3965460628"}]: dispatch 2024-04-02T15:31:45.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:45 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/3075918570' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:6800/3965460628"}]': finished 2024-04-02T15:31:45.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:45 smithi106 ceph-mon[25843]: osdmap e84: 8 total, 8 up, 8 in 2024-04-02T15:31:45.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:45 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/4157910006' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/4051190714"}]: dispatch 2024-04-02T15:31:45.870 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:31:45 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:31:45] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:31:45.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:45 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/3075918570' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:6800/3965460628"}]': finished 2024-04-02T15:31:45.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:45 smithi087 ceph-mon[29707]: osdmap e84: 8 total, 8 up, 8 in 2024-04-02T15:31:45.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:45 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/4157910006' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/4051190714"}]: dispatch 2024-04-02T15:31:45.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:45 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/3075918570' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:6800/3965460628"}]': finished 2024-04-02T15:31:45.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:45 smithi087 ceph-mon[24900]: osdmap e84: 8 total, 8 up, 8 in 2024-04-02T15:31:45.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:45 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/4157910006' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/4051190714"}]: dispatch 2024-04-02T15:31:46.620 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:31:46 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:31:46] "GET /metrics HTTP/1.1" 200 207981 "" "Prometheus/2.33.4" 2024-04-02T15:31:46.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:46 smithi087 ceph-mon[29707]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:31:46.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:46 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/4157910006' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/4051190714"}]': finished 2024-04-02T15:31:46.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:46 smithi087 ceph-mon[29707]: osdmap e85: 8 total, 8 up, 8 in 2024-04-02T15:31:46.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:46 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:46.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:46 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:46.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:46 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:31:46.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:46 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:31:46.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:46 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:31:46.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:46 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:46.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:46 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/102852450' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:6800/2380456996"}]: dispatch 2024-04-02T15:31:46.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:46 smithi087 ceph-mon[24900]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:31:46.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:46 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/4157910006' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/4051190714"}]': finished 2024-04-02T15:31:46.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:46 smithi087 ceph-mon[24900]: osdmap e85: 8 total, 8 up, 8 in 2024-04-02T15:31:46.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:46 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:46.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:46 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:46.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:46 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:31:46.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:46 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:31:46.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:46 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:31:46.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:46 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:46.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:46 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/102852450' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:6800/2380456996"}]: dispatch 2024-04-02T15:31:46.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:46 smithi106 ceph-mon[25843]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:31:46.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:46 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/4157910006' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/4051190714"}]': finished 2024-04-02T15:31:46.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:46 smithi106 ceph-mon[25843]: osdmap e85: 8 total, 8 up, 8 in 2024-04-02T15:31:46.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:46 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:46.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:46 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:46.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:46 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:31:46.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:46 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:31:46.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:46 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:31:46.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:46 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:46.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:46 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/102852450' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:6800/2380456996"}]: dispatch 2024-04-02T15:31:47.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:47 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/102852450' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:6800/2380456996"}]': finished 2024-04-02T15:31:47.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:47 smithi106 ceph-mon[25843]: osdmap e86: 8 total, 8 up, 8 in 2024-04-02T15:31:47.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:47 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/409648484' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/2150387812"}]: dispatch 2024-04-02T15:31:47.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:47 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/102852450' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:6800/2380456996"}]': finished 2024-04-02T15:31:47.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:47 smithi087 ceph-mon[24900]: osdmap e86: 8 total, 8 up, 8 in 2024-04-02T15:31:47.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:47 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/409648484' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/2150387812"}]: dispatch 2024-04-02T15:31:47.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:47 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/102852450' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:6800/2380456996"}]': finished 2024-04-02T15:31:47.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:47 smithi087 ceph-mon[29707]: osdmap e86: 8 total, 8 up, 8 in 2024-04-02T15:31:47.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:47 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/409648484' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/2150387812"}]: dispatch 2024-04-02T15:31:48.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:48 smithi106 ceph-mon[25843]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:31:48.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:48 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/409648484' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/2150387812"}]': finished 2024-04-02T15:31:48.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:48 smithi106 ceph-mon[25843]: osdmap e87: 8 total, 8 up, 8 in 2024-04-02T15:31:48.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:48 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/233176165' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/2076084295"}]: dispatch 2024-04-02T15:31:48.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:48 smithi106 ceph-mon[25843]: from='client.? ' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/2076084295"}]: dispatch 2024-04-02T15:31:48.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:48 smithi087 ceph-mon[24900]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:31:48.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:48 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/409648484' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/2150387812"}]': finished 2024-04-02T15:31:48.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:48 smithi087 ceph-mon[24900]: osdmap e87: 8 total, 8 up, 8 in 2024-04-02T15:31:48.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:48 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/233176165' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/2076084295"}]: dispatch 2024-04-02T15:31:48.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:48 smithi087 ceph-mon[24900]: from='client.? ' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/2076084295"}]: dispatch 2024-04-02T15:31:48.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:48 smithi087 ceph-mon[29707]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:31:48.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:48 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/409648484' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/2150387812"}]': finished 2024-04-02T15:31:48.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:48 smithi087 ceph-mon[29707]: osdmap e87: 8 total, 8 up, 8 in 2024-04-02T15:31:48.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:48 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/233176165' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/2076084295"}]: dispatch 2024-04-02T15:31:48.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:48 smithi087 ceph-mon[29707]: from='client.? ' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/2076084295"}]: dispatch 2024-04-02T15:31:49.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:49 smithi106 ceph-mon[25843]: from='client.? ' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/2076084295"}]': finished 2024-04-02T15:31:49.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:49 smithi106 ceph-mon[25843]: osdmap e88: 8 total, 8 up, 8 in 2024-04-02T15:31:49.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:49 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/1602342595' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:6801/2380456996"}]: dispatch 2024-04-02T15:31:49.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:49 smithi106 ceph-mon[25843]: from='client.? ' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:6801/2380456996"}]: dispatch 2024-04-02T15:31:49.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:49 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:31:49.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:49 smithi087 ceph-mon[24900]: from='client.? ' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/2076084295"}]': finished 2024-04-02T15:31:49.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:49 smithi087 ceph-mon[24900]: osdmap e88: 8 total, 8 up, 8 in 2024-04-02T15:31:49.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:49 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/1602342595' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:6801/2380456996"}]: dispatch 2024-04-02T15:31:49.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:49 smithi087 ceph-mon[24900]: from='client.? ' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:6801/2380456996"}]: dispatch 2024-04-02T15:31:49.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:49 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:31:49.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:49 smithi087 ceph-mon[29707]: from='client.? ' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/2076084295"}]': finished 2024-04-02T15:31:49.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:49 smithi087 ceph-mon[29707]: osdmap e88: 8 total, 8 up, 8 in 2024-04-02T15:31:49.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:49 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/1602342595' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:6801/2380456996"}]: dispatch 2024-04-02T15:31:49.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:49 smithi087 ceph-mon[29707]: from='client.? ' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:6801/2380456996"}]: dispatch 2024-04-02T15:31:49.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:49 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:31:50.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:50 smithi106 ceph-mon[25843]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:31:50.868 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:50 smithi106 ceph-mon[25843]: from='client.? ' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:6801/2380456996"}]': finished 2024-04-02T15:31:50.868 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:50 smithi106 ceph-mon[25843]: osdmap e89: 8 total, 8 up, 8 in 2024-04-02T15:31:50.868 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:50 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/1806253531' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:6801/3965460628"}]: dispatch 2024-04-02T15:31:50.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:50 smithi087 ceph-mon[24900]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:31:50.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:50 smithi087 ceph-mon[24900]: from='client.? ' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:6801/2380456996"}]': finished 2024-04-02T15:31:50.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:50 smithi087 ceph-mon[24900]: osdmap e89: 8 total, 8 up, 8 in 2024-04-02T15:31:50.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:50 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/1806253531' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:6801/3965460628"}]: dispatch 2024-04-02T15:31:50.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:50 smithi087 ceph-mon[29707]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:31:50.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:50 smithi087 ceph-mon[29707]: from='client.? ' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:6801/2380456996"}]': finished 2024-04-02T15:31:50.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:50 smithi087 ceph-mon[29707]: osdmap e89: 8 total, 8 up, 8 in 2024-04-02T15:31:50.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:50 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/1806253531' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:6801/3965460628"}]: dispatch 2024-04-02T15:31:51.371 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:31:51 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:31:51] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:31:51.837 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:51 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/1806253531' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:6801/3965460628"}]': finished 2024-04-02T15:31:51.837 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:51 smithi106 ceph-mon[25843]: osdmap e90: 8 total, 8 up, 8 in 2024-04-02T15:31:51.838 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:51 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/1113319211' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/3045002547"}]: dispatch 2024-04-02T15:31:51.838 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:51 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:51.838 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:51 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:51.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:51 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/1806253531' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:6801/3965460628"}]': finished 2024-04-02T15:31:51.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:51 smithi087 ceph-mon[29707]: osdmap e90: 8 total, 8 up, 8 in 2024-04-02T15:31:51.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:51 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/1113319211' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/3045002547"}]: dispatch 2024-04-02T15:31:51.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:51 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:51.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:51 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:51.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:51 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/1806253531' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:6801/3965460628"}]': finished 2024-04-02T15:31:51.873 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:51 smithi087 ceph-mon[24900]: osdmap e90: 8 total, 8 up, 8 in 2024-04-02T15:31:51.873 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:51 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/1113319211' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/3045002547"}]: dispatch 2024-04-02T15:31:51.873 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:51 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:51.873 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:51 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:31:52.786 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:52 smithi106 ceph-mon[25843]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:31:52.786 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:52 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/1113319211' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/3045002547"}]': finished 2024-04-02T15:31:52.786 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:52 smithi106 ceph-mon[25843]: osdmap e91: 8 total, 8 up, 8 in 2024-04-02T15:31:52.786 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:52 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/1221466178' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/2722718326"}]: dispatch 2024-04-02T15:31:52.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:52 smithi087 ceph-mon[29707]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:31:52.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:52 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/1113319211' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/3045002547"}]': finished 2024-04-02T15:31:52.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:52 smithi087 ceph-mon[29707]: osdmap e91: 8 total, 8 up, 8 in 2024-04-02T15:31:52.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:52 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/1221466178' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/2722718326"}]: dispatch 2024-04-02T15:31:52.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:52 smithi087 ceph-mon[24900]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:31:52.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:52 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/1113319211' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/3045002547"}]': finished 2024-04-02T15:31:52.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:52 smithi087 ceph-mon[24900]: osdmap e91: 8 total, 8 up, 8 in 2024-04-02T15:31:52.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:52 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/1221466178' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/2722718326"}]: dispatch 2024-04-02T15:31:53.730 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:53 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/1221466178' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/2722718326"}]': finished 2024-04-02T15:31:53.730 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:53 smithi106 ceph-mon[25843]: osdmap e92: 8 total, 8 up, 8 in 2024-04-02T15:31:53.730 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:53 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/1020076742' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:6801/2108248912"}]: dispatch 2024-04-02T15:31:53.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:53 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/1221466178' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/2722718326"}]': finished 2024-04-02T15:31:53.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:53 smithi087 ceph-mon[29707]: osdmap e92: 8 total, 8 up, 8 in 2024-04-02T15:31:53.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:53 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/1020076742' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:6801/2108248912"}]: dispatch 2024-04-02T15:31:53.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:53 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/1221466178' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/2722718326"}]': finished 2024-04-02T15:31:53.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:53 smithi087 ceph-mon[24900]: osdmap e92: 8 total, 8 up, 8 in 2024-04-02T15:31:53.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:53 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/1020076742' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:6801/2108248912"}]: dispatch 2024-04-02T15:31:54.048 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:31:53 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:31:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:31:54.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:54 smithi106 ceph-mon[25843]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:31:54.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:54 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/1020076742' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:6801/2108248912"}]': finished 2024-04-02T15:31:54.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:54 smithi106 ceph-mon[25843]: osdmap e93: 8 total, 8 up, 8 in 2024-04-02T15:31:54.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:54 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/1843356629' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/3328279098"}]: dispatch 2024-04-02T15:31:54.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:54 smithi106 ceph-mon[25843]: from='client.? ' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/3328279098"}]: dispatch 2024-04-02T15:31:54.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:54 smithi087 ceph-mon[29707]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:31:54.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:54 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/1020076742' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:6801/2108248912"}]': finished 2024-04-02T15:31:54.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:54 smithi087 ceph-mon[29707]: osdmap e93: 8 total, 8 up, 8 in 2024-04-02T15:31:54.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:54 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/1843356629' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/3328279098"}]: dispatch 2024-04-02T15:31:54.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:54 smithi087 ceph-mon[29707]: from='client.? ' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/3328279098"}]: dispatch 2024-04-02T15:31:54.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:54 smithi087 ceph-mon[24900]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:31:54.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:54 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/1020076742' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:6801/2108248912"}]': finished 2024-04-02T15:31:54.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:54 smithi087 ceph-mon[24900]: osdmap e93: 8 total, 8 up, 8 in 2024-04-02T15:31:54.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:54 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/1843356629' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/3328279098"}]: dispatch 2024-04-02T15:31:54.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:54 smithi087 ceph-mon[24900]: from='client.? ' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/3328279098"}]: dispatch 2024-04-02T15:31:55.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:55 smithi106 ceph-mon[25843]: from='client.? ' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/3328279098"}]': finished 2024-04-02T15:31:55.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:55 smithi106 ceph-mon[25843]: osdmap e94: 8 total, 8 up, 8 in 2024-04-02T15:31:55.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:55 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/995757736' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/593333980"}]: dispatch 2024-04-02T15:31:55.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:55 smithi087 ceph-mon[29707]: from='client.? ' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/3328279098"}]': finished 2024-04-02T15:31:55.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:55 smithi087 ceph-mon[29707]: osdmap e94: 8 total, 8 up, 8 in 2024-04-02T15:31:55.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:55 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/995757736' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/593333980"}]: dispatch 2024-04-02T15:31:55.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:55 smithi087 ceph-mon[24900]: from='client.? ' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/3328279098"}]': finished 2024-04-02T15:31:55.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:55 smithi087 ceph-mon[24900]: osdmap e94: 8 total, 8 up, 8 in 2024-04-02T15:31:55.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:55 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/995757736' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/593333980"}]: dispatch 2024-04-02T15:31:56.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:31:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:31:56] "GET /metrics HTTP/1.1" 200 208045 "" "Prometheus/2.33.4" 2024-04-02T15:31:56.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:56 smithi087 ceph-mon[29707]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:31:56.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:56 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/995757736' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/593333980"}]': finished 2024-04-02T15:31:56.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:56 smithi087 ceph-mon[29707]: osdmap e95: 8 total, 8 up, 8 in 2024-04-02T15:31:56.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:56 smithi087 ceph-mon[24900]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:31:56.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:56 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/995757736' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/593333980"}]': finished 2024-04-02T15:31:56.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:56 smithi087 ceph-mon[24900]: osdmap e95: 8 total, 8 up, 8 in 2024-04-02T15:31:56.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:56 smithi106 ceph-mon[25843]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:31:56.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:56 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/995757736' entity='client.iscsi.foo.smithi087.icnqbx' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.87:0/593333980"}]': finished 2024-04-02T15:31:56.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:56 smithi106 ceph-mon[25843]: osdmap e95: 8 total, 8 up, 8 in 2024-04-02T15:31:58.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:58 smithi106 ceph-mon[25843]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T15:31:58.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:58 smithi087 ceph-mon[29707]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T15:31:58.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:58 smithi087 ceph-mon[24900]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T15:31:59.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:31:59 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:31:59.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:31:59 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:31:59.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:31:59 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:32:00.730 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:00 smithi087 ceph-mon[29707]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:00.731 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:00 smithi087 ceph-mon[24900]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:00.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:00 smithi106 ceph-mon[25843]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:02.035 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:32:01 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:32:01] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:32:02.777 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:02 smithi087 ceph-mon[29707]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-02T15:32:02.777 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:02 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:32:02.778 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:02 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:32:02.778 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:02 smithi087 ceph-mon[24900]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-02T15:32:02.778 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:02 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:32:02.778 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:02 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:32:03.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:02 smithi106 ceph-mon[25843]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-02T15:32:03.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:02 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:32:03.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:02 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:32:04.116 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:32:03 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:32:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:32:05.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:04 smithi106 ceph-mon[25843]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T15:32:05.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:04 smithi087 ceph-mon[29707]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T15:32:05.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:04 smithi087 ceph-mon[24900]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-02T15:32:06.620 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:32:06 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:32:06] "GET /metrics HTTP/1.1" 200 208051 "" "Prometheus/2.33.4" 2024-04-02T15:32:07.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:06 smithi106 ceph-mon[25843]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T15:32:07.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:06 smithi087 ceph-mon[29707]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T15:32:07.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:06 smithi087 ceph-mon[24900]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-02T15:32:09.099 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:08 smithi087 ceph-mon[29707]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-02T15:32:09.099 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:08 smithi087 ceph-mon[24900]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-02T15:32:09.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:08 smithi106 ceph-mon[25843]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-02T15:32:10.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:09 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:32:10.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:09 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:32:10.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:09 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:32:11.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:10 smithi106 ceph-mon[25843]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:11.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:10 smithi087 ceph-mon[29707]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:11.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:10 smithi087 ceph-mon[24900]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:11.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:32:11 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:32:11] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:32:12.471 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:12 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:32:12.471 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:12 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:32:12.471 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:12 smithi106 ceph-mon[25843]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:32:12.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:12 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:32:12.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:12 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:32:12.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:12 smithi087 ceph-mon[29707]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:32:12.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:12 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:32:12.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:12 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:32:12.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:12 smithi087 ceph-mon[24900]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:32:13.955 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:32:13 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:32:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:32:14.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:14 smithi106 ceph-mon[25843]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:14.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:14 smithi087 ceph-mon[29707]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:14.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:14 smithi087 ceph-mon[24900]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:16.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:16 smithi106 ceph-mon[25843]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:16.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:16 smithi087 ceph-mon[29707]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:16.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:16 smithi087 ceph-mon[24900]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:16.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:32:16 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:32:16] "GET /metrics HTTP/1.1" 200 208051 "" "Prometheus/2.33.4" 2024-04-02T15:32:18.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:18 smithi106 ceph-mon[25843]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:32:18.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:18 smithi087 ceph-mon[29707]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:32:18.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:18 smithi087 ceph-mon[24900]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:32:19.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:19 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:32:19.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:19 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:32:19.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:19 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:32:20.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:20 smithi087 ceph-mon[24900]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:20.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:20 smithi087 ceph-mon[29707]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:20.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:20 smithi106 ceph-mon[25843]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:21.951 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:32:21 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:32:21] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:32:23.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:22 smithi106 ceph-mon[25843]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:32:23.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:22 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:32:23.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:22 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:32:23.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:22 smithi087 ceph-mon[24900]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:32:23.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:22 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:32:23.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:22 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:32:23.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:22 smithi087 ceph-mon[29707]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:32:23.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:22 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:32:23.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:22 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:32:24.018 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:32:23 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:32:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:32:25.029 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:24 smithi106 ceph-mon[25843]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:25.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:24 smithi087 ceph-mon[24900]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:25.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:24 smithi087 ceph-mon[29707]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:26.620 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:32:26 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:32:26] "GET /metrics HTTP/1.1" 200 208029 "" "Prometheus/2.33.4" 2024-04-02T15:32:27.072 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:26 smithi087 ceph-mon[29707]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:27.072 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:26 smithi087 ceph-mon[24900]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:27.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:26 smithi106 ceph-mon[25843]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:29.114 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:28 smithi087 ceph-mon[24900]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:32:29.115 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:28 smithi087 ceph-mon[29707]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:32:29.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:28 smithi106 ceph-mon[25843]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:32:30.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:29 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:32:30.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:29 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:32:30.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:29 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:32:31.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:30 smithi106 ceph-mon[25843]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:31.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:30 smithi087 ceph-mon[29707]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:31.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:30 smithi087 ceph-mon[24900]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:31.620 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:32:31 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:32:31] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:32:32.495 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:32 smithi106 ceph-mon[25843]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:32:32.495 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:32 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:32:32.495 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:32 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:32:32.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:32 smithi087 ceph-mon[24900]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:32:32.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:32 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:32:32.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:32 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:32:32.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:32 smithi087 ceph-mon[29707]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:32:32.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:32 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:32:32.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:32 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:32:33.866 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:32:33 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:32:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:32:33.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:32:33 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:32:33.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:32:33.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:32:33 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:32:33.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.106 because it doesn't contain any IP SANs" 2024-04-02T15:32:34.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:34 smithi106 ceph-mon[25843]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:34.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:34 smithi087 ceph-mon[29707]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:34.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:34 smithi087 ceph-mon[24900]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:35.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:35 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:32:35.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:35 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:32:35.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:35 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:32:35.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:35 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:32:35.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:35 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:32:35.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:35 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:32:36.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:36 smithi106 ceph-mon[25843]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:36.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:36 smithi087 ceph-mon[29707]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:36.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:36 smithi087 ceph-mon[24900]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:36.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:32:36 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:32:36] "GET /metrics HTTP/1.1" 200 207980 "" "Prometheus/2.33.4" 2024-04-02T15:32:38.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:38 smithi106 ceph-mon[25843]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:32:38.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:38 smithi087 ceph-mon[29707]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:32:38.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:38 smithi087 ceph-mon[24900]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:32:39.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:39 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:32:39.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:39 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:32:39.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:39 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:32:40.608 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:40 smithi087 ceph-mon[29707]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:40.608 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:40 smithi087 ceph-mon[24900]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:40.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:40 smithi106 ceph-mon[25843]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:41.874 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:32:41 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:32:41] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:32:42.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:42 smithi087 ceph-mon[24900]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:32:42.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:42 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:32:42.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:42 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:32:42.872 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:42 smithi087 ceph-mon[29707]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:32:42.872 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:42 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:32:42.872 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:42 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:32:43.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:42 smithi106 ceph-mon[25843]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:32:43.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:42 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:32:43.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:42 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:32:43.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:32:43 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:32:43.506Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.106 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:32:43.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:32:43 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:32:43.508Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:32:43.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:32:43 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:32:43.508Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.106 because it doesn't contain any IP SANs" 2024-04-02T15:32:44.117 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:32:43 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:32:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:32:45.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:44 smithi106 ceph-mon[25843]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:45.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:44 smithi087 ceph-mon[29707]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:45.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:44 smithi087 ceph-mon[24900]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:46.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:45 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:32:46.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:45 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:32:46.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:45 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:32:46.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:45 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:32:46.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:45 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:32:46.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:45 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:32:46.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:45 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:32:46.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:45 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:32:46.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:45 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:32:46.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:45 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:32:46.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:45 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:32:46.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:45 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:32:46.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:32:46 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:32:46] "GET /metrics HTTP/1.1" 200 207980 "" "Prometheus/2.33.4" 2024-04-02T15:32:47.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:46 smithi106 ceph-mon[25843]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:47.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:46 smithi087 ceph-mon[24900]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:47.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:46 smithi087 ceph-mon[29707]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:49.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:48 smithi106 ceph-mon[25843]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:32:49.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:48 smithi087 ceph-mon[29707]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:32:49.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:48 smithi087 ceph-mon[24900]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:32:50.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:49 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:32:50.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:49 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:32:50.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:49 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:32:51.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:50 smithi106 ceph-mon[25843]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:51.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:50 smithi087 ceph-mon[24900]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:51.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:50 smithi087 ceph-mon[29707]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:51.871 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:32:51 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:32:51] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:32:52.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:52 smithi106 ceph-mon[25843]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:32:52.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:52 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:32:52.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:52 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:32:52.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:52 smithi087 ceph-mon[29707]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:32:52.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:52 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:32:52.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:52 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:32:52.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:52 smithi087 ceph-mon[24900]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:32:52.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:52 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:32:52.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:52 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:32:53.870 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:32:53 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:32:53.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.106 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:32:53.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:32:53 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:32:53.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:32:53.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:32:53 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:32:53.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.106 because it doesn't contain any IP SANs" 2024-04-02T15:32:54.116 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:32:53 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:32:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:32:54.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:54 smithi106 ceph-mon[25843]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:54.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:54 smithi087 ceph-mon[29707]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:54.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:54 smithi087 ceph-mon[24900]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:56.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:56 smithi087 ceph-mon[24900]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:56.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:56 smithi087 ceph-mon[29707]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:56.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:32:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:32:56] "GET /metrics HTTP/1.1" 200 207986 "" "Prometheus/2.33.4" 2024-04-02T15:32:56.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:56 smithi106 ceph-mon[25843]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:32:58.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:58 smithi106 ceph-mon[25843]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:32:58.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:58 smithi087 ceph-mon[29707]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:32:58.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:58 smithi087 ceph-mon[24900]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:32:59.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:32:59 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:32:59.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:32:59 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:32:59.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:32:59 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:33:00.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:00 smithi106 ceph-mon[25843]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:00.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:00 smithi087 ceph-mon[29707]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:00.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:00 smithi087 ceph-mon[24900]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:02.246 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:33:01 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:33:01] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:33:03.341 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:02 smithi087 ceph-mon[24900]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:33:03.341 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:02 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:33:03.341 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:02 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:33:03.341 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:02 smithi087 ceph-mon[29707]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:33:03.341 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:02 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:33:03.341 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:02 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:33:03.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:02 smithi106 ceph-mon[25843]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:33:03.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:02 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:33:03.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:02 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:33:03.621 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:33:03 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:33:03.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.106 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:33:03.621 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:33:03 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:33:03.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:33:03.621 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:33:03 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:33:03.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.106 because it doesn't contain any IP SANs" 2024-04-02T15:33:04.117 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:33:03 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:33:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:33:05.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:04 smithi106 ceph-mon[25843]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:05.372 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:04 smithi087 ceph-mon[29707]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:05.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:04 smithi087 ceph-mon[24900]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:06.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:33:06 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:33:06] "GET /metrics HTTP/1.1" 200 207982 "" "Prometheus/2.33.4" 2024-04-02T15:33:07.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:06 smithi106 ceph-mon[25843]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:07.370 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:06 smithi087 ceph-mon[24900]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:07.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:06 smithi087 ceph-mon[29707]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:09.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:08 smithi106 ceph-mon[25843]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:33:09.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:08 smithi087 ceph-mon[24900]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:33:09.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:08 smithi087 ceph-mon[29707]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:33:10.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:10 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:33:10.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:10 smithi106 ceph-mon[25843]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:10.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:10 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:33:10.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:10 smithi087 ceph-mon[29707]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:10.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:10 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:33:10.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:10 smithi087 ceph-mon[24900]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:11.871 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:33:11 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:33:11] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:33:12.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:12 smithi106 ceph-mon[25843]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:33:12.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:12 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:33:12.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:12 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:33:12.870 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:12 smithi087 ceph-mon[29707]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:33:12.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:12 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:33:12.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:12 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:33:12.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:12 smithi087 ceph-mon[24900]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:33:12.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:12 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:33:12.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:12 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:33:13.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:33:13 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:33:13.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.106 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:33:13.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:33:13 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:33:13.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:33:13.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:33:13 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:33:13.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.106 because it doesn't contain any IP SANs" 2024-04-02T15:33:14.091 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:33:13 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:33:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:33:14.762 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:14 smithi106 ceph-mon[25843]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:14.870 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:14 smithi087 ceph-mon[24900]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:14.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:14 smithi087 ceph-mon[29707]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:16.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:16 smithi106 ceph-mon[25843]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:16.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:16 smithi087 ceph-mon[29707]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:16.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:16 smithi087 ceph-mon[24900]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:16.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:33:16 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:33:16] "GET /metrics HTTP/1.1" 200 207982 "" "Prometheus/2.33.4" 2024-04-02T15:33:18.792 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:18 smithi087 ceph-mon[29707]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:33:18.792 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:18 smithi087 ceph-mon[24900]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:33:18.866 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:18 smithi106 ceph-mon[25843]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:33:20.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:20 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:33:20.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:20 smithi106 ceph-mon[25843]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:20.620 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:20 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:33:20.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:20 smithi087 ceph-mon[24900]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:20.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:20 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:33:20.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:20 smithi087 ceph-mon[29707]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:22.357 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:33:22 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:33:22] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:33:23.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:23 smithi106 ceph-mon[25843]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:33:23.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:23 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:33:23.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:23 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:33:23.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:23 smithi087 ceph-mon[29707]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:33:23.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:23 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:33:23.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:23 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:33:23.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:23 smithi087 ceph-mon[24900]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:33:23.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:23 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:33:23.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:23 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:33:23.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:33:23 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:33:23.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.106 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:33:23.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:33:23 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:33:23.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:33:23.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:33:23 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:33:23.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.106 because it doesn't contain any IP SANs" 2024-04-02T15:33:24.118 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:33:23 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:33:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:33:25.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:25 smithi087 ceph-mon[24900]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:25.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:25 smithi087 ceph-mon[29707]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:25.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:25 smithi106 ceph-mon[25843]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:26.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:33:26 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:33:26] "GET /metrics HTTP/1.1" 200 207991 "" "Prometheus/2.33.4" 2024-04-02T15:33:27.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:27 smithi106 ceph-mon[25843]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:27.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:27 smithi087 ceph-mon[29707]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:27.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:27 smithi087 ceph-mon[24900]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:29.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:29 smithi106 ceph-mon[25843]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:33:29.370 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:29 smithi087 ceph-mon[29707]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:33:29.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:29 smithi087 ceph-mon[24900]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:33:30.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:30 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:33:30.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:30 smithi106 ceph-mon[25843]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-02T15:33:30.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:30 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:33:30.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:30 smithi087 ceph-mon[29707]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-02T15:33:30.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:30 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:33:30.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:30 smithi087 ceph-mon[24900]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-02T15:33:32.121 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:33:31 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:33:31] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:33:33.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:32 smithi106 ceph-mon[25843]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:33:33.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:32 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:33:33.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:32 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:33:33.120 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:32 smithi087 ceph-mon[29707]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:33:33.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:32 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:33:33.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:32 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:33:33.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:32 smithi087 ceph-mon[24900]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:33:33.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:32 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:33:33.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:32 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:33:33.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:33:33 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:33:33.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.106 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:33:33.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:33:33 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:33:33.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:33:33.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:33:33 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:33:33.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.106 because it doesn't contain any IP SANs" 2024-04-02T15:33:34.097 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:33:33 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:33:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:33:34.758 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:34 smithi106 ceph-mon[25843]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-02T15:33:34.927 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:34 smithi087 ceph-mon[29707]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-02T15:33:34.927 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:34 smithi087 ceph-mon[24900]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-02T15:33:35.823 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:35 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:33:35.823 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:35 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:33:35.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:35 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:33:35.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:35 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:33:35.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:35 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:33:35.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:35 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:33:36.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:33:36 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:33:36] "GET /metrics HTTP/1.1" 200 207980 "" "Prometheus/2.33.4" 2024-04-02T15:33:37.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:36 smithi106 ceph-mon[25843]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-02T15:33:37.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:36 smithi087 ceph-mon[29707]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-02T15:33:37.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:36 smithi087 ceph-mon[24900]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-02T15:33:39.021 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:38 smithi087 ceph-mon[29707]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:33:39.021 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:38 smithi087 ceph-mon[24900]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:33:39.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:38 smithi106 ceph-mon[25843]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:33:40.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:40 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:33:40.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:40 smithi106 ceph-mon[25843]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:40.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:40 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:33:40.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:40 smithi087 ceph-mon[24900]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:40.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:40 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:33:40.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:40 smithi087 ceph-mon[29707]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:42.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:33:42 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:33:42] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:33:43.247 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:43 smithi087 ceph-mon[29707]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:33:43.248 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:43 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:33:43.248 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:43 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:33:43.248 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:43 smithi087 ceph-mon[24900]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:33:43.248 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:43 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:33:43.248 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:43 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:33:43.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:43 smithi106 ceph-mon[25843]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:33:43.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:43 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:33:43.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:43 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:33:43.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:33:43 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:33:43.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.106 because it doesn't contain any IP SANs" 2024-04-02T15:33:43.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:33:43 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:33:43.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:33:43.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:33:43 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:33:43.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.106 because it doesn't contain any IP SANs" 2024-04-02T15:33:44.117 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:33:43 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:33:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:33:45.266 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:45 smithi087 ceph-mon[29707]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:45.266 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:45 smithi087 ceph-mon[24900]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:45.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:45 smithi106 ceph-mon[25843]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:46.380 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:33:46 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:33:46] "GET /metrics HTTP/1.1" 200 207980 "" "Prometheus/2.33.4" 2024-04-02T15:33:46.381 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:46 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:33:46.381 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:46 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:33:46.381 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:46 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:33:46.381 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:46 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:33:46.381 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:46 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:33:46.381 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:46 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:33:46.381 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:46 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:33:46.382 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:46 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:33:46.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:46 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:33:46.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:46 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:33:46.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:46 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:33:46.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:46 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:33:47.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:47 smithi106 ceph-mon[25843]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:47.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:47 smithi087 ceph-mon[24900]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:47.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:47 smithi087 ceph-mon[29707]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:48.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:48 smithi106 ceph-mon[25843]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:33:48.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:48 smithi087 ceph-mon[29707]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:33:48.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:48 smithi087 ceph-mon[24900]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:33:50.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:50 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:33:50.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:50 smithi106 ceph-mon[25843]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:50.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:50 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:33:50.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:50 smithi087 ceph-mon[29707]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:50.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:50 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:33:50.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:50 smithi087 ceph-mon[24900]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:52.122 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:33:51 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:33:51] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:33:53.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:52 smithi106 ceph-mon[25843]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:33:53.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:52 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:33:53.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:52 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:33:53.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:52 smithi087 ceph-mon[29707]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:33:53.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:52 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:33:53.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:52 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:33:53.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:52 smithi087 ceph-mon[24900]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:33:53.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:52 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:33:53.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:52 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:33:53.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:33:53 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:33:53.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.106 because it doesn't contain any IP SANs" 2024-04-02T15:33:53.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:33:53 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:33:53.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:33:53.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:33:53 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:33:53.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.106 because it doesn't contain any IP SANs" 2024-04-02T15:33:54.018 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:33:53 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:33:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:33:54.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:54 smithi106 ceph-mon[25843]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:55.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:54 smithi087 ceph-mon[29707]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:55.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:54 smithi087 ceph-mon[24900]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:56.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:33:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:33:56] "GET /metrics HTTP/1.1" 200 207982 "" "Prometheus/2.33.4" 2024-04-02T15:33:57.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:56 smithi106 ceph-mon[25843]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:57.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:56 smithi087 ceph-mon[29707]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:57.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:56 smithi087 ceph-mon[24900]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:33:59.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:33:58 smithi106 ceph-mon[25843]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:33:59.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:33:58 smithi087 ceph-mon[24900]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:33:59.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:33:58 smithi087 ceph-mon[29707]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:34:00.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:00 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:34:00.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:00 smithi106 ceph-mon[25843]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:00.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:00 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:34:00.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:00 smithi087 ceph-mon[29707]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:00.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:00 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:34:00.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:00 smithi087 ceph-mon[24900]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:02.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:02 smithi106 ceph-mon[25843]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:34:02.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:02 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:02.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:02 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:02.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:34:02 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:34:02] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:34:02.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:02 smithi087 ceph-mon[24900]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:34:02.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:02 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:02.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:02 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:02.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:02 smithi087 ceph-mon[29707]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:34:02.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:02 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:02.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:02 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:03.797 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:34:03 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:34:03.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.106 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:34:03.797 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:34:03 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:34:03.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:34:03.797 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:34:03 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:34:03.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.106 because it doesn't contain any IP SANs" 2024-04-02T15:34:04.117 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:34:03 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:34:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:34:04.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:04 smithi087 ceph-mon[29707]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:04.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:04 smithi087 ceph-mon[24900]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:04.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:04 smithi106 ceph-mon[25843]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:06.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:06 smithi106 ceph-mon[25843]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:06.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:06 smithi087 ceph-mon[29707]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:06.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:34:06 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:34:06] "GET /metrics HTTP/1.1" 200 207952 "" "Prometheus/2.33.4" 2024-04-02T15:34:06.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:06 smithi087 ceph-mon[24900]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:08.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:08 smithi106 ceph-mon[25843]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:34:08.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:08 smithi087 ceph-mon[29707]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:34:08.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:08 smithi087 ceph-mon[24900]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:34:10.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:10 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:34:10.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:10 smithi106 ceph-mon[25843]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:10.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:10 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:34:10.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:10 smithi087 ceph-mon[29707]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:10.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:10 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:34:10.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:10 smithi087 ceph-mon[24900]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:12.121 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:34:11 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:34:11] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:34:13.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:12 smithi106 ceph-mon[25843]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:34:13.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:12 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:13.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:12 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:13.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:12 smithi087 ceph-mon[29707]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:34:13.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:12 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:13.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:12 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:13.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:12 smithi087 ceph-mon[24900]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:34:13.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:12 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:13.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:12 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:13.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:34:13 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:34:13.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.106 because it doesn't contain any IP SANs" 2024-04-02T15:34:13.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:34:13 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:34:13.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:34:13.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:34:13 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:34:13.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.106 because it doesn't contain any IP SANs" 2024-04-02T15:34:14.117 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:34:13 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:34:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:34:15.097 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:14 smithi106 ceph-mon[25843]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:15.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:14 smithi087 ceph-mon[29707]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:15.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:14 smithi087 ceph-mon[24900]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:16.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:34:16 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:34:16] "GET /metrics HTTP/1.1" 200 207952 "" "Prometheus/2.33.4" 2024-04-02T15:34:17.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:16 smithi106 ceph-mon[25843]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:17.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:16 smithi087 ceph-mon[24900]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:17.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:16 smithi087 ceph-mon[29707]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:19.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:18 smithi106 ceph-mon[25843]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:34:19.120 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:18 smithi087 ceph-mon[24900]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:34:19.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:18 smithi087 ceph-mon[29707]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:34:20.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:20 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:34:20.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:20 smithi106 ceph-mon[25843]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:20.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:20 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:34:20.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:20 smithi087 ceph-mon[29707]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:20.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:20 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:34:20.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:20 smithi087 ceph-mon[24900]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:22.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:22 smithi106 ceph-mon[25843]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:34:22.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:22 smithi087 ceph-mon[29707]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:34:22.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:22 smithi087 ceph-mon[24900]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:34:22.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:34:22 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:34:22] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:34:23.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:23 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:23.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:23 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:23.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:23 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:23.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:23 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:23.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:23 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:23.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:23 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:23.621 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:34:23 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:34:23.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.106 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:34:23.621 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:34:23 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:34:23.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:34:23.621 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:34:23 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:34:23.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.106 because it doesn't contain any IP SANs" 2024-04-02T15:34:24.116 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:34:23 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:34:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:34:24.530 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:24 smithi106 ceph-mon[25843]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:24.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:24 smithi087 ceph-mon[29707]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:24.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:24 smithi087 ceph-mon[24900]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:26.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:26 smithi106 ceph-mon[25843]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:26.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:34:26 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:34:26] "GET /metrics HTTP/1.1" 200 207942 "" "Prometheus/2.33.4" 2024-04-02T15:34:26.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:26 smithi087 ceph-mon[29707]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:26.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:26 smithi087 ceph-mon[24900]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:28.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:28 smithi106 ceph-mon[25843]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:34:28.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:28 smithi087 ceph-mon[24900]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:34:28.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:28 smithi087 ceph-mon[29707]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:34:30.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:30 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:34:30.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:30 smithi106 ceph-mon[25843]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:30.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:30 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:34:30.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:30 smithi087 ceph-mon[29707]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:30.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:30 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:34:30.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:30 smithi087 ceph-mon[24900]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:32.371 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:34:31 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:34:31] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:34:33.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:32 smithi106 ceph-mon[25843]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:34:33.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:32 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:33.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:32 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:33.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:32 smithi087 ceph-mon[24900]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:34:33.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:32 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:33.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:32 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:33.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:32 smithi087 ceph-mon[29707]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:34:33.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:32 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:33.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:32 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:33.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:34:33 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:34:33.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.106 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:34:33.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:34:33 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:34:33.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:34:33.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:34:33 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:34:33.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.106 because it doesn't contain any IP SANs" 2024-04-02T15:34:34.117 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:34:33 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:34:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:34:35.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:34 smithi106 ceph-mon[25843]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:35.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:34 smithi087 ceph-mon[24900]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:35.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:34 smithi087 ceph-mon[29707]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:36.275 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:35 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:34:36.275 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:35 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:34:36.326 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:35 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:34:36.327 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:35 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:34:36.327 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:35 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:34:36.327 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:35 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:34:36.469 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-04-02T15:34:36.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:34:36 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:34:36] "GET /metrics HTTP/1.1" 200 207952 "" "Prometheus/2.33.4" 2024-04-02T15:34:37.068 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:36 smithi087 ceph-mon[29707]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:37.068 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:36 smithi087 ceph-mon[24900]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:37.269 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:36 smithi106 ceph-mon[25843]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:39.003 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-04-02T15:34:39.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:38 smithi106 ceph-mon[25843]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:34:39.240 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:38 smithi087 ceph-mon[29707]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:34:39.240 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:38 smithi087 ceph-mon[24900]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:34:40.346 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:40 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:34:40.346 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:40 smithi087 ceph-mon[24900]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:40.346 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:40 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:34:40.346 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:40 smithi087 ceph-mon[29707]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:40.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:40 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:34:40.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:40 smithi106 ceph-mon[25843]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:41.464 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph config set global log_to_journald false --force' 2024-04-02T15:34:42.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:42 smithi106 ceph-mon[25843]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:34:42.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:42 smithi087 ceph-mon[29707]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:34:42.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:42 smithi087 ceph-mon[24900]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:34:42.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:34:42 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:34:42] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:34:43.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:43 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:43.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:43 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:43.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:43 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:43.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:43 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:43.622 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:34:43 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:34:43.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.106 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:34:43.622 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:34:43 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:34:43.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:34:43.622 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:34:43 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:34:43.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.106 because it doesn't contain any IP SANs" 2024-04-02T15:34:43.730 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:43 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:43.730 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:43 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:43.961 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph orch ps' 2024-04-02T15:34:44.117 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:34:43 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:34:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:34:44.557 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:44 smithi087 ceph-mon[29707]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:44.557 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:44 smithi087 ceph-mon[24900]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:44.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:44 smithi106 ceph-mon[25843]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:46.454 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:34:46 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:34:46] "GET /metrics HTTP/1.1" 200 207952 "" "Prometheus/2.33.4" 2024-04-02T15:34:46.713 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:46 smithi087 ceph-mon[24900]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:46.713 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:46 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:34:46.713 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:46 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:34:46.713 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:46 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:34:46.713 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:46 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:46.714 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:46 smithi087 ceph-mon[29707]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:46.714 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:46 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:34:46.714 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:46 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:34:46.714 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:46 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:34:46.714 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:46 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:46.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:46 smithi106 ceph-mon[25843]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:46.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:46 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:34:46.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:46 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:34:46.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:46 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:34:46.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:46 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:46.915 INFO:teuthology.orchestra.run.smithi087.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T15:34:46.915 INFO:teuthology.orchestra.run.smithi087.stdout:agent.smithi087 smithi087 running 4s ago 9m - - 2024-04-02T15:34:46.915 INFO:teuthology.orchestra.run.smithi087.stdout:agent.smithi106 smithi106 running 14s ago 9m - - 2024-04-02T15:34:46.915 INFO:teuthology.orchestra.run.smithi087.stdout:alertmanager.a smithi087 *:9093,9094 running (3m) 4s ago 3m 28.3M - ba2b418f427c 93850beda829 2024-04-02T15:34:46.915 INFO:teuthology.orchestra.run.smithi087.stdout:grafana.a smithi106 *:3000 running (3m) 14s ago 3m 55.5M - 8.3.5 dad864ee21e9 8956d795b0b4 2024-04-02T15:34:46.915 INFO:teuthology.orchestra.run.smithi087.stdout:iscsi.foo.smithi087.icnqbx smithi087 running (3m) 4s ago 3m 52.9M - 3.5 e1d6a67b021e 8278ac6ea2a5 2024-04-02T15:34:46.915 INFO:teuthology.orchestra.run.smithi087.stdout:mgr.x smithi106 *:8443 running (8m) 14s ago 8m 415M - 17.2.0 e1d6a67b021e 7d9b02b111e7 2024-04-02T15:34:46.915 INFO:teuthology.orchestra.run.smithi087.stdout:mgr.y smithi087 *:9283 running (10m) 4s ago 10m 465M - 17.2.0 e1d6a67b021e 5d422ff3cc7d 2024-04-02T15:34:46.915 INFO:teuthology.orchestra.run.smithi087.stdout:mon.a smithi087 running (10m) 4s ago 10m 49.3M 2048M 17.2.0 e1d6a67b021e e28fd32a7d9a 2024-04-02T15:34:46.915 INFO:teuthology.orchestra.run.smithi087.stdout:mon.b smithi106 running (8m) 14s ago 8m 43.5M 2048M 17.2.0 e1d6a67b021e d9de5ac6a73e 2024-04-02T15:34:46.915 INFO:teuthology.orchestra.run.smithi087.stdout:mon.c smithi087 running (9m) 4s ago 9m 41.9M 2048M 17.2.0 e1d6a67b021e 40f23934618c 2024-04-02T15:34:46.915 INFO:teuthology.orchestra.run.smithi087.stdout:node-exporter.a smithi087 *:9100 running (4m) 4s ago 4m 17.9M - 1dbe0e931976 feba1ed50125 2024-04-02T15:34:46.915 INFO:teuthology.orchestra.run.smithi087.stdout:node-exporter.b smithi106 *:9100 running (4m) 14s ago 4m 14.7M - 1dbe0e931976 fb10fb2b50d2 2024-04-02T15:34:46.915 INFO:teuthology.orchestra.run.smithi087.stdout:osd.0 smithi087 running (8m) 4s ago 8m 49.5M 4096M 17.2.0 e1d6a67b021e 644fbe6ff2bc 2024-04-02T15:34:46.916 INFO:teuthology.orchestra.run.smithi087.stdout:osd.1 smithi087 running (7m) 4s ago 7m 49.1M 4096M 17.2.0 e1d6a67b021e 8996e84013c2 2024-04-02T15:34:46.916 INFO:teuthology.orchestra.run.smithi087.stdout:osd.2 smithi087 running (7m) 4s ago 7m 47.4M 4096M 17.2.0 e1d6a67b021e 944a519c2f4b 2024-04-02T15:34:46.916 INFO:teuthology.orchestra.run.smithi087.stdout:osd.3 smithi087 running (7m) 4s ago 7m 46.6M 4096M 17.2.0 e1d6a67b021e 00c03ac366b2 2024-04-02T15:34:46.916 INFO:teuthology.orchestra.run.smithi087.stdout:osd.4 smithi106 running (6m) 14s ago 6m 48.4M 4096M 17.2.0 e1d6a67b021e c96f419b11f6 2024-04-02T15:34:46.916 INFO:teuthology.orchestra.run.smithi087.stdout:osd.5 smithi106 running (6m) 14s ago 6m 48.5M 4096M 17.2.0 e1d6a67b021e 91062a17779c 2024-04-02T15:34:46.916 INFO:teuthology.orchestra.run.smithi087.stdout:osd.6 smithi106 running (5m) 14s ago 5m 47.6M 4096M 17.2.0 e1d6a67b021e f582842222f2 2024-04-02T15:34:46.916 INFO:teuthology.orchestra.run.smithi087.stdout:osd.7 smithi106 running (5m) 14s ago 5m 48.2M 4096M 17.2.0 e1d6a67b021e 85ed81d59bdd 2024-04-02T15:34:46.916 INFO:teuthology.orchestra.run.smithi087.stdout:prometheus.a smithi106 *:9095 running (3m) 14s ago 4m 67.0M - 514e6a882f6e 3ea0ba4743fb 2024-04-02T15:34:46.916 INFO:teuthology.orchestra.run.smithi087.stdout:rgw.foo.smithi087.awfxzs smithi087 *:8000 running (3m) 4s ago 3m 72.1M - 17.2.0 e1d6a67b021e ecaa755cb095 2024-04-02T15:34:46.916 INFO:teuthology.orchestra.run.smithi087.stdout:rgw.foo.smithi106.bfeikp smithi106 *:8000 running (3m) 14s ago 3m 71.9M - 17.2.0 e1d6a67b021e a6dc1247c030 2024-04-02T15:34:47.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:47 smithi087 ceph-mon[29707]: from='client.14973 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:34:47.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:47 smithi087 ceph-mon[24900]: from='client.14973 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:34:47.768 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph versions' 2024-04-02T15:34:47.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:47 smithi106 ceph-mon[25843]: from='client.14973 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:34:48.715 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:48 smithi087 ceph-mon[29707]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:34:48.716 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:48 smithi087 ceph-mon[24900]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:34:48.723 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:48 smithi106 ceph-mon[25843]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:34:50.387 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-02T15:34:50.387 INFO:teuthology.orchestra.run.smithi087.stdout: "mon": { 2024-04-02T15:34:50.387 INFO:teuthology.orchestra.run.smithi087.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-02T15:34:50.388 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-02T15:34:50.388 INFO:teuthology.orchestra.run.smithi087.stdout: "mgr": { 2024-04-02T15:34:50.388 INFO:teuthology.orchestra.run.smithi087.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-02T15:34:50.388 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-02T15:34:50.388 INFO:teuthology.orchestra.run.smithi087.stdout: "osd": { 2024-04-02T15:34:50.388 INFO:teuthology.orchestra.run.smithi087.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-02T15:34:50.388 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-02T15:34:50.388 INFO:teuthology.orchestra.run.smithi087.stdout: "mds": {}, 2024-04-02T15:34:50.388 INFO:teuthology.orchestra.run.smithi087.stdout: "rgw": { 2024-04-02T15:34:50.388 INFO:teuthology.orchestra.run.smithi087.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-02T15:34:50.388 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-02T15:34:50.388 INFO:teuthology.orchestra.run.smithi087.stdout: "overall": { 2024-04-02T15:34:50.388 INFO:teuthology.orchestra.run.smithi087.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 15 2024-04-02T15:34:50.389 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-02T15:34:50.389 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-02T15:34:50.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:50 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:34:50.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:50 smithi106 ceph-mon[25843]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:50.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:50 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:34:50.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:50 smithi087 ceph-mon[29707]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:50.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:50 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:34:50.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:50 smithi087 ceph-mon[24900]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:51.106 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph -s' 2024-04-02T15:34:51.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:51 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/2576059453' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T15:34:51.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:51 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/2576059453' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T15:34:51.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:51 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/2576059453' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T15:34:52.311 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:34:52 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:34:52] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:34:53.053 INFO:teuthology.orchestra.run.smithi087.stdout: cluster: 2024-04-02T15:34:53.053 INFO:teuthology.orchestra.run.smithi087.stdout: id: 0400842c-f105-11ee-b647-cb9ed24678a4 2024-04-02T15:34:53.053 INFO:teuthology.orchestra.run.smithi087.stdout: health: HEALTH_OK 2024-04-02T15:34:53.053 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-02T15:34:53.053 INFO:teuthology.orchestra.run.smithi087.stdout: services: 2024-04-02T15:34:53.053 INFO:teuthology.orchestra.run.smithi087.stdout: mon: 3 daemons, quorum a,c,b (age 8m) 2024-04-02T15:34:53.054 INFO:teuthology.orchestra.run.smithi087.stdout: mgr: y(active, since 5m), standbys: x 2024-04-02T15:34:53.054 INFO:teuthology.orchestra.run.smithi087.stdout: osd: 8 osds: 8 up (since 5m), 8 in (since 5m) 2024-04-02T15:34:53.054 INFO:teuthology.orchestra.run.smithi087.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-04-02T15:34:53.054 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-02T15:34:53.054 INFO:teuthology.orchestra.run.smithi087.stdout: data: 2024-04-02T15:34:53.054 INFO:teuthology.orchestra.run.smithi087.stdout: pools: 6 pools, 161 pgs 2024-04-02T15:34:53.054 INFO:teuthology.orchestra.run.smithi087.stdout: objects: 209 objects, 585 KiB 2024-04-02T15:34:53.054 INFO:teuthology.orchestra.run.smithi087.stdout: usage: 63 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:34:53.054 INFO:teuthology.orchestra.run.smithi087.stdout: pgs: 161 active+clean 2024-04-02T15:34:53.054 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-02T15:34:53.054 INFO:teuthology.orchestra.run.smithi087.stdout: io: 2024-04-02T15:34:53.054 INFO:teuthology.orchestra.run.smithi087.stdout: client: 1.2 KiB/s rd, 1 op/s rd, 0 op/s wr 2024-04-02T15:34:53.055 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-02T15:34:53.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:53 smithi106 ceph-mon[25843]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:34:53.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:53 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:53.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:53 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:53.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:53 smithi087 ceph-mon[29707]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:34:53.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:53 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:53.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:53 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:53.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:53 smithi087 ceph-mon[24900]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:34:53.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:53 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:53.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:53 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:53.727 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:34:53 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:34:53.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.106 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:34:53.727 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:34:53 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:34:53.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:34:53.727 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:34:53 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:34:53.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.106 because it doesn't contain any IP SANs" 2024-04-02T15:34:53.754 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph orch ls' 2024-04-02T15:34:54.056 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:34:53 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:34:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:34:54.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:54 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/1034771249' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-02T15:34:54.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:54 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/1034771249' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-02T15:34:54.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:54 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/1034771249' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-02T15:34:55.323 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:55 smithi087 ceph-mon[29707]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:55.323 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:55 smithi087 ceph-mon[24900]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:55.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:55 smithi106 ceph-mon[25843]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:55.729 INFO:teuthology.orchestra.run.smithi087.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-04-02T15:34:55.729 INFO:teuthology.orchestra.run.smithi087.stdout:agent 2/2 13s ago 5m * 2024-04-02T15:34:55.729 INFO:teuthology.orchestra.run.smithi087.stdout:alertmanager ?:9093,9094 1/1 13s ago 5m smithi087=a;count:1 2024-04-02T15:34:55.730 INFO:teuthology.orchestra.run.smithi087.stdout:grafana ?:3000 1/1 3s ago 5m smithi106=a;count:1 2024-04-02T15:34:55.730 INFO:teuthology.orchestra.run.smithi087.stdout:iscsi.foo 1/1 13s ago 3m count:1 2024-04-02T15:34:55.730 INFO:teuthology.orchestra.run.smithi087.stdout:mgr 2/2 13s ago 8m smithi087=y;smithi106=x;count:2 2024-04-02T15:34:55.730 INFO:teuthology.orchestra.run.smithi087.stdout:mon 3/3 13s ago 9m smithi087:172.21.15.87=a;smithi087:[v2:172.21.15.87:3301,v1:172.21.15.87:6790]=c;smithi106:172.21.15.106=b;count:3 2024-04-02T15:34:55.730 INFO:teuthology.orchestra.run.smithi087.stdout:node-exporter ?:9100 2/2 13s ago 5m smithi087=a;smithi106=b;count:2 2024-04-02T15:34:55.730 INFO:teuthology.orchestra.run.smithi087.stdout:osd 8 13s ago - 2024-04-02T15:34:55.730 INFO:teuthology.orchestra.run.smithi087.stdout:prometheus ?:9095 1/1 3s ago 5m smithi106=a;count:1 2024-04-02T15:34:55.730 INFO:teuthology.orchestra.run.smithi087.stdout:rgw.foo ?:8000 2/2 13s ago 3m count:2 2024-04-02T15:34:56.371 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:34:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:34:56] "GET /metrics HTTP/1.1" 200 207937 "" "Prometheus/2.33.4" 2024-04-02T15:34:56.455 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-02T15:34:57.256 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:57 smithi106 ceph-mon[25843]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:57.256 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:57 smithi106 ceph-mon[25843]: from='client.24892 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:34:57.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:57 smithi087 ceph-mon[29707]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:57.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:57 smithi087 ceph-mon[29707]: from='client.24892 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:34:57.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:57 smithi087 ceph-mon[24900]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:34:57.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:57 smithi087 ceph-mon[24900]: from='client.24892 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:34:58.982 INFO:teuthology.orchestra.run.smithi087.stdout:Scheduled to redeploy mgr.x on host 'smithi106' 2024-04-02T15:34:59.260 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:59 smithi087 ceph-mon[29707]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:34:59.260 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:59 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/3624398788' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-02T15:34:59.260 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:59 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:59.260 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:34:59 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:34:59.260 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:59 smithi087 ceph-mon[24900]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:34:59.260 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:59 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/3624398788' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-02T15:34:59.260 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:59 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:59.260 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:34:59 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:34:59.335 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:59 smithi106 ceph-mon[25843]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:34:59.336 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:59 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/3624398788' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-02T15:34:59.336 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:59 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:34:59.336 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:34:59 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:34:59.694 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph orch ps --refresh' 2024-04-02T15:35:00.334 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:00 smithi087 ceph-mon[29707]: from='client.15003 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:3f3f3ed992429b85be729743ad3b39d52dd90744", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:35:00.334 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:00 smithi087 ceph-mon[29707]: Schedule redeploy daemon mgr.x 2024-04-02T15:35:00.335 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:00 smithi087 ceph-mon[24900]: from='client.15003 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:3f3f3ed992429b85be729743ad3b39d52dd90744", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:35:00.335 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:00 smithi087 ceph-mon[24900]: Schedule redeploy daemon mgr.x 2024-04-02T15:35:00.366 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:00 smithi106 ceph-mon[25843]: from='client.15003 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:3f3f3ed992429b85be729743ad3b39d52dd90744", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:35:00.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:00 smithi106 ceph-mon[25843]: Schedule redeploy daemon mgr.x 2024-04-02T15:35:01.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:01 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:35:01.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:01 smithi106 ceph-mon[25843]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:01.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:01 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:35:01.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:01 smithi087 ceph-mon[29707]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:01.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:01 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:35:01.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:01 smithi087 ceph-mon[24900]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:01.838 INFO:teuthology.orchestra.run.smithi087.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T15:35:01.838 INFO:teuthology.orchestra.run.smithi087.stdout:agent.smithi087 smithi087 running 19s ago 9m - - 2024-04-02T15:35:01.838 INFO:teuthology.orchestra.run.smithi087.stdout:agent.smithi106 smithi106 running 9s ago 9m - - 2024-04-02T15:35:01.839 INFO:teuthology.orchestra.run.smithi087.stdout:alertmanager.a smithi087 *:9093,9094 running (3m) 19s ago 4m 28.3M - ba2b418f427c 93850beda829 2024-04-02T15:35:01.839 INFO:teuthology.orchestra.run.smithi087.stdout:grafana.a smithi106 *:3000 running (3m) 9s ago 3m 55.2M - 8.3.5 dad864ee21e9 8956d795b0b4 2024-04-02T15:35:01.839 INFO:teuthology.orchestra.run.smithi087.stdout:iscsi.foo.smithi087.icnqbx smithi087 running (3m) 19s ago 3m 52.9M - 3.5 e1d6a67b021e 8278ac6ea2a5 2024-04-02T15:35:01.839 INFO:teuthology.orchestra.run.smithi087.stdout:mgr.x smithi106 *:8443 running (8m) 9s ago 8m 415M - 17.2.0 e1d6a67b021e 7d9b02b111e7 2024-04-02T15:35:01.839 INFO:teuthology.orchestra.run.smithi087.stdout:mgr.y smithi087 *:9283 running (10m) 19s ago 10m 465M - 17.2.0 e1d6a67b021e 5d422ff3cc7d 2024-04-02T15:35:01.839 INFO:teuthology.orchestra.run.smithi087.stdout:mon.a smithi087 running (10m) 19s ago 10m 49.3M 2048M 17.2.0 e1d6a67b021e e28fd32a7d9a 2024-04-02T15:35:01.839 INFO:teuthology.orchestra.run.smithi087.stdout:mon.b smithi106 running (9m) 9s ago 9m 40.1M 2048M 17.2.0 e1d6a67b021e d9de5ac6a73e 2024-04-02T15:35:01.839 INFO:teuthology.orchestra.run.smithi087.stdout:mon.c smithi087 running (9m) 19s ago 9m 41.9M 2048M 17.2.0 e1d6a67b021e 40f23934618c 2024-04-02T15:35:01.839 INFO:teuthology.orchestra.run.smithi087.stdout:node-exporter.a smithi087 *:9100 running (5m) 19s ago 5m 17.9M - 1dbe0e931976 feba1ed50125 2024-04-02T15:35:01.839 INFO:teuthology.orchestra.run.smithi087.stdout:node-exporter.b smithi106 *:9100 running (5m) 9s ago 5m 14.3M - 1dbe0e931976 fb10fb2b50d2 2024-04-02T15:35:01.839 INFO:teuthology.orchestra.run.smithi087.stdout:osd.0 smithi087 running (8m) 19s ago 8m 49.5M 4096M 17.2.0 e1d6a67b021e 644fbe6ff2bc 2024-04-02T15:35:01.839 INFO:teuthology.orchestra.run.smithi087.stdout:osd.1 smithi087 running (8m) 19s ago 8m 49.1M 4096M 17.2.0 e1d6a67b021e 8996e84013c2 2024-04-02T15:35:01.839 INFO:teuthology.orchestra.run.smithi087.stdout:osd.2 smithi087 running (7m) 19s ago 7m 47.4M 4096M 17.2.0 e1d6a67b021e 944a519c2f4b 2024-04-02T15:35:01.839 INFO:teuthology.orchestra.run.smithi087.stdout:osd.3 smithi087 running (7m) 19s ago 7m 46.6M 4096M 17.2.0 e1d6a67b021e 00c03ac366b2 2024-04-02T15:35:01.840 INFO:teuthology.orchestra.run.smithi087.stdout:osd.4 smithi106 running (6m) 9s ago 6m 48.6M 4096M 17.2.0 e1d6a67b021e c96f419b11f6 2024-04-02T15:35:01.840 INFO:teuthology.orchestra.run.smithi087.stdout:osd.5 smithi106 running (6m) 9s ago 6m 48.7M 4096M 17.2.0 e1d6a67b021e 91062a17779c 2024-04-02T15:35:01.840 INFO:teuthology.orchestra.run.smithi087.stdout:osd.6 smithi106 running (6m) 9s ago 6m 47.8M 4096M 17.2.0 e1d6a67b021e f582842222f2 2024-04-02T15:35:01.840 INFO:teuthology.orchestra.run.smithi087.stdout:osd.7 smithi106 running (5m) 9s ago 5m 48.4M 4096M 17.2.0 e1d6a67b021e 85ed81d59bdd 2024-04-02T15:35:01.840 INFO:teuthology.orchestra.run.smithi087.stdout:prometheus.a smithi106 *:9095 running (3m) 9s ago 5m 67.1M - 514e6a882f6e 3ea0ba4743fb 2024-04-02T15:35:01.840 INFO:teuthology.orchestra.run.smithi087.stdout:rgw.foo.smithi087.awfxzs smithi087 *:8000 running (3m) 19s ago 3m 72.1M - 17.2.0 e1d6a67b021e ecaa755cb095 2024-04-02T15:35:01.840 INFO:teuthology.orchestra.run.smithi087.stdout:rgw.foo.smithi106.bfeikp smithi106 *:8000 running (3m) 9s ago 3m 72.0M - 17.2.0 e1d6a67b021e a6dc1247c030 2024-04-02T15:35:02.681 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'sleep 180' 2024-04-02T15:35:02.871 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:35:02 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:35:02] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:35:03.259 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:03 smithi087 ceph-mon[24900]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:35:03.259 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:03 smithi087 ceph-mon[24900]: from='client.24910 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:35:03.259 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:03 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:35:03.259 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:03 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:35:03.260 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:03 smithi087 ceph-mon[29707]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:35:03.260 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:03 smithi087 ceph-mon[29707]: from='client.24910 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:35:03.260 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:03 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:35:03.260 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:03 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:35:03.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:03 smithi106 ceph-mon[25843]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:35:03.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:03 smithi106 ceph-mon[25843]: from='client.24910 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:35:03.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:03 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:35:03.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:03 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:35:03.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:35:03 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:35:03.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.106 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:35:03.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:35:03 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:35:03.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:35:03.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:35:03 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:35:03.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.106 because it doesn't contain any IP SANs" 2024-04-02T15:35:04.117 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:35:03 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:35:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:35:05.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:05 smithi106 ceph-mon[25843]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:05.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:05 smithi087 ceph-mon[29707]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:05.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:05 smithi087 ceph-mon[24900]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:06.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:35:06 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:35:06] "GET /metrics HTTP/1.1" 200 207940 "" "Prometheus/2.33.4" 2024-04-02T15:35:07.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:07 smithi106 ceph-mon[25843]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:07.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:07 smithi087 ceph-mon[29707]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:07.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:07 smithi087 ceph-mon[24900]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:09.269 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:09 smithi087 ceph-mon[29707]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:35:09.270 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:09 smithi087 ceph-mon[24900]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:35:09.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:09 smithi106 ceph-mon[25843]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:35:10.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:10 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:35:10.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:10 smithi106 ceph-mon[25843]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:10.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:10 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:35:10.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:10 smithi087 ceph-mon[29707]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:10.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:10 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:35:10.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:10 smithi087 ceph-mon[24900]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:12.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:35:12 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:35:12] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:35:13.515 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:13 smithi087 ceph-mon[29707]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:35:13.515 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:13 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:35:13.515 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:13 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:35:13.515 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:13 smithi087 ceph-mon[24900]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:35:13.515 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:13 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:35:13.515 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:13 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:35:13.616 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:13 smithi106 ceph-mon[25843]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:35:13.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:13 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:35:13.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:13 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:35:13.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:35:13 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:35:13.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.106 because it doesn't contain any IP SANs" 2024-04-02T15:35:14.117 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:35:13 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:35:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:35:15.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:15 smithi106 ceph-mon[25843]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:15.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:15 smithi087 ceph-mon[24900]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:15.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:15 smithi087 ceph-mon[29707]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:16.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:35:16 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:35:16] "GET /metrics HTTP/1.1" 200 207940 "" "Prometheus/2.33.4" 2024-04-02T15:35:17.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:17 smithi106 ceph-mon[25843]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:17.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:17 smithi087 ceph-mon[29707]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:17.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:17 smithi087 ceph-mon[24900]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:19.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:19 smithi106 ceph-mon[25843]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:35:19.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:19 smithi087 ceph-mon[29707]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:35:19.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:19 smithi087 ceph-mon[24900]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:35:20.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:20 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:35:20.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:20 smithi106 ceph-mon[25843]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:20.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:20 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:35:20.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:20 smithi087 ceph-mon[24900]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:20.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:20 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:35:20.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:20 smithi087 ceph-mon[29707]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:22.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:22 smithi106 ceph-mon[25843]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:35:22.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:22 smithi087 ceph-mon[29707]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:35:22.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:22 smithi087 ceph-mon[24900]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:35:23.121 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:35:22 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:35:22] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:35:24.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:23 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:35:24.118 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:23 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:35:24.118 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:35:23 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:35:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:35:24.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:23 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:35:24.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:23 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:35:24.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:23 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:35:24.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:23 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:35:25.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:24 smithi106 ceph-mon[25843]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:25.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:24 smithi087 ceph-mon[29707]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:25.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:24 smithi087 ceph-mon[24900]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:26.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:35:26 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:35:26] "GET /metrics HTTP/1.1" 200 207942 "" "Prometheus/2.33.4" 2024-04-02T15:35:27.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:26 smithi106 ceph-mon[25843]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:27.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:26 smithi087 ceph-mon[29707]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:27.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:26 smithi087 ceph-mon[24900]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:28.872 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:28 smithi087 ceph-mon[29707]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:35:28.872 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:28 smithi087 ceph-mon[24900]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:35:29.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:28 smithi106 ceph-mon[25843]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:35:30.609 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:30 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:35:30.609 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:30 smithi106 ceph-mon[25843]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:30.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:30 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:35:30.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:30 smithi087 ceph-mon[29707]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:30.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:30 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:35:30.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:30 smithi087 ceph-mon[24900]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:32.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:32 smithi106 ceph-mon[25843]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:35:32.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:32 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:35:32.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:32 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:35:32.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:35:32 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:35:32] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:35:32.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:32 smithi087 ceph-mon[29707]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:35:32.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:32 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:35:32.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:32 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:35:32.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:32 smithi087 ceph-mon[24900]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:35:32.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:32 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:35:32.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:32 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:35:34.117 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:35:33 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:35:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:35:34.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:34 smithi106 ceph-mon[25843]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:34.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:34 smithi087 ceph-mon[29707]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:34.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:34 smithi087 ceph-mon[24900]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:35.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:35 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:35:35.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:35 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:35:35.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:35 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:35:35.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:35 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:35:35.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:35 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:35:35.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:35 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:35:36.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:36 smithi087 ceph-mon[29707]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:36.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:36 smithi087 ceph-mon[24900]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:36.622 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:35:36 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:35:36] "GET /metrics HTTP/1.1" 200 207941 "" "Prometheus/2.33.4" 2024-04-02T15:35:36.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:36 smithi106 ceph-mon[25843]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:38.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:38 smithi087 ceph-mon[29707]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:35:38.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:38 smithi087 ceph-mon[24900]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:35:38.700 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:38 smithi106 ceph-mon[25843]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:35:40.547 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:40 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:35:40.547 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:40 smithi106 ceph-mon[25843]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:40.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:40 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:35:40.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:40 smithi087 ceph-mon[29707]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:40.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:40 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:35:40.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:40 smithi087 ceph-mon[24900]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:42.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:42 smithi106 ceph-mon[25843]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:35:42.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:42 smithi087 ceph-mon[29707]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:35:42.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:42 smithi087 ceph-mon[24900]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:35:43.121 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:35:42 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:35:42] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:35:44.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:43 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:35:44.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:43 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:35:44.117 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:35:43 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:35:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:35:44.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:43 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:35:44.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:43 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:35:44.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:43 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:35:44.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:43 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:35:45.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:44 smithi106 ceph-mon[25843]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:45.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:44 smithi087 ceph-mon[29707]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:45.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:44 smithi087 ceph-mon[24900]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:46.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:35:46 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:35:46] "GET /metrics HTTP/1.1" 200 207941 "" "Prometheus/2.33.4" 2024-04-02T15:35:46.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:46 smithi106 ceph-mon[25843]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:47.030 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:46 smithi087 ceph-mon[29707]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:47.031 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:46 smithi087 ceph-mon[24900]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:49.006 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:48 smithi087 ceph-mon[24900]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:35:49.006 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:48 smithi087 ceph-mon[29707]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:35:49.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:48 smithi106 ceph-mon[25843]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:35:50.540 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:50 smithi087 ceph-mon[29707]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:50.540 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:50 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:35:50.540 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:50 smithi087 ceph-mon[24900]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:50.540 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:50 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:35:50.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:50 smithi106 ceph-mon[25843]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:50.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:50 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:35:52.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:52 smithi106 ceph-mon[25843]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:35:52.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:52 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:35:52.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:52 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:35:52.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:35:52 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:35:52] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:35:52.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:52 smithi087 ceph-mon[29707]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:35:52.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:52 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:35:52.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:52 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:35:52.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:52 smithi087 ceph-mon[24900]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:35:52.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:52 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:35:52.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:52 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:35:54.117 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:35:53 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:35:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:35:54.519 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:54 smithi106 ceph-mon[25843]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:54.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:54 smithi087 ceph-mon[29707]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:54.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:54 smithi087 ceph-mon[24900]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:56.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:56 smithi106 ceph-mon[25843]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:56.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:56 smithi087 ceph-mon[29707]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:56.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:56 smithi087 ceph-mon[24900]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:35:56.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:35:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:35:56] "GET /metrics HTTP/1.1" 200 207940 "" "Prometheus/2.33.4" 2024-04-02T15:35:58.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:35:58 smithi106 ceph-mon[25843]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:35:58.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:35:58 smithi087 ceph-mon[29707]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:35:58.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:35:58 smithi087 ceph-mon[24900]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:36:00.514 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:00 smithi106 ceph-mon[25843]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:00.514 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:00 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:36:00.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:00 smithi087 ceph-mon[29707]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:00.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:00 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:36:00.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:00 smithi087 ceph-mon[24900]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:00.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:00 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:36:02.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:02 smithi106 ceph-mon[25843]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:36:02.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:02 smithi087 ceph-mon[29707]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:36:02.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:02 smithi087 ceph-mon[24900]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:36:03.121 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:36:02 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:36:02] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:36:04.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:03 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:04.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:03 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:04.117 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:03 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:36:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:36:04.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:03 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:04.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:03 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:04.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:03 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:04.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:03 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:05.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:04 smithi106 ceph-mon[25843]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:05.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:04 smithi087 ceph-mon[29707]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:05.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:04 smithi087 ceph-mon[24900]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:06.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:36:06 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:36:06] "GET /metrics HTTP/1.1" 200 207938 "" "Prometheus/2.33.4" 2024-04-02T15:36:07.116 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:06 smithi087 ceph-mon[29707]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:07.117 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:06 smithi087 ceph-mon[24900]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:07.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:06 smithi106 ceph-mon[25843]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:09.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:08 smithi106 ceph-mon[25843]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:36:09.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:08 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:09.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:08 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:36:09.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:08 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:36:09.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:08 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:09.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:08 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:09.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:08 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:09.118 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:08 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T15:36:09.118 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:08 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T15:36:09.118 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:08 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:36:09.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:08 smithi087 ceph-mon[29707]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:36:09.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:08 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:09.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:08 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:36:09.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:08 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:36:09.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:08 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:09.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:08 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:09.122 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:08 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:09.122 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:08 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T15:36:09.122 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:08 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T15:36:09.122 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:08 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:36:09.122 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:08 smithi087 ceph-mon[24900]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:36:09.122 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:08 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:09.122 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:08 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:36:09.122 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:08 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:36:09.122 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:08 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:09.122 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:08 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:09.122 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:08 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:09.122 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:08 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T15:36:09.123 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:08 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T15:36:09.123 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:08 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:36:10.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:09 smithi106 ceph-mon[25843]: Deploying daemon mgr.x on smithi106 2024-04-02T15:36:10.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:09 smithi087 ceph-mon[29707]: Deploying daemon mgr.x on smithi106 2024-04-02T15:36:10.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:09 smithi087 ceph-mon[24900]: Deploying daemon mgr.x on smithi106 2024-04-02T15:36:11.100 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:10 smithi087 ceph-mon[29707]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:11.100 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:10 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:36:11.100 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:10 smithi087 ceph-mon[24900]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:11.100 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:10 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:36:11.116 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:10 smithi106 ceph-mon[25843]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:11.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:10 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:36:12.849 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:36:12 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:36:12] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:36:13.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:12 smithi106 ceph-mon[25843]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:36:13.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:12 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:13.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:12 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:13.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:12 smithi087 ceph-mon[29707]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:36:13.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:12 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:13.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:12 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:13.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:12 smithi087 ceph-mon[24900]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:36:13.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:12 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:13.122 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:12 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:14.117 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:13 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:36:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:36:15.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:14 smithi106 ceph-mon[25843]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:15.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:14 smithi087 ceph-mon[29707]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:15.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:14 smithi087 ceph-mon[24900]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:16.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:36:16 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:36:16] "GET /metrics HTTP/1.1" 200 207938 "" "Prometheus/2.33.4" 2024-04-02T15:36:16.873 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:16 smithi087 ceph-mon[29707]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:16.874 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:16 smithi087 ceph-mon[24900]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:17.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:16 smithi106 ceph-mon[25843]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:19.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:18 smithi106 ceph-mon[25843]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:36:19.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:18 smithi087 ceph-mon[29707]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:36:19.121 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:18 smithi087 ceph-mon[24900]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:36:20.477 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:20 smithi106 ceph-mon[25843]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:20.477 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:20 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:36:20.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:20 smithi087 ceph-mon[24900]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:20.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:20 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:36:20.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:20 smithi087 ceph-mon[29707]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:20.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:20 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:36:22.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:22 smithi087 ceph-mon[29707]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:36:22.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:22 smithi087 ceph-mon[24900]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:36:22.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:22 smithi106 ceph-mon[25843]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:36:23.371 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:36:22 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:36:22] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:36:23.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:36:23 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:36:23.507Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.106 because it doesn't contain any IP SANs" 2024-04-02T15:36:23.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:36:23 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:36:23.507Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:36:24.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:23 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:24.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:23 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:24.117 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:23 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:36:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:36:24.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:23 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:24.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:23 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:24.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:23 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:24.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:23 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:25.351 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:24 smithi106 ceph-mon[25843]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:25.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:24 smithi087 ceph-mon[29707]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:25.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:24 smithi087 ceph-mon[24900]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:26.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:36:26 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:36:26] "GET /metrics HTTP/1.1" 200 207936 "" "Prometheus/2.33.4" 2024-04-02T15:36:27.297 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:26 smithi087 ceph-mon[29707]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:27.297 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:26 smithi087 ceph-mon[24900]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:27.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:26 smithi106 ceph-mon[25843]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:29.263 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:28 smithi087 ceph-mon[29707]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:36:29.263 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:28 smithi087 ceph-mon[24900]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:36:29.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:28 smithi106 ceph-mon[25843]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:36:30.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:30 smithi106 ceph-mon[25843]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:30.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:30 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:36:30.617 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:30 smithi087 ceph-mon[29707]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:30.617 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:30 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:36:30.618 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:30 smithi087 ceph-mon[24900]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:30.618 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:30 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:36:32.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:32 smithi087 ceph-mon[24900]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:36:32.622 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:36:32 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:36:32] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:36:32.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:32 smithi087 ceph-mon[29707]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:36:32.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:32 smithi106 ceph-mon[25843]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:36:33.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:33 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:33.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:33 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:33.867 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:33 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:36:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:36:33.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:33 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:33.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:33 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:33.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:33 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:33.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:33 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:33.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:36:33 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:36:33.505Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.106 because it doesn't contain any IP SANs" 2024-04-02T15:36:33.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:36:33 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:36:33.507Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:36:33.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:36:33 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:36:33.508Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.106 because it doesn't contain any IP SANs" 2024-04-02T15:36:34.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:34 smithi106 ceph-mon[25843]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:34.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:34 smithi087 ceph-mon[24900]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:34.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:34 smithi087 ceph-mon[29707]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:35.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:35 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:36:35.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:35 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:36:35.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:35 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:36:35.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:35 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:36:35.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:35 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:36:35.872 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:35 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:36:36.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:36 smithi087 ceph-mon[29707]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:36.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:36 smithi087 ceph-mon[24900]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:36.622 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:36:36 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:36:36] "GET /metrics HTTP/1.1" 200 207930 "" "Prometheus/2.33.4" 2024-04-02T15:36:36.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:36 smithi106 ceph-mon[25843]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:38.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:38 smithi106 ceph-mon[25843]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:36:38.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:38 smithi087 ceph-mon[29707]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:36:38.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:38 smithi087 ceph-mon[24900]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:36:40.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:40 smithi106 ceph-mon[25843]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:40.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:40 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:36:40.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:40 smithi087 ceph-mon[29707]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:40.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:40 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:36:40.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:40 smithi087 ceph-mon[24900]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:40.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:40 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:36:42.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:42 smithi106 ceph-mon[25843]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:36:42.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:42 smithi087 ceph-mon[29707]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:36:42.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:42 smithi087 ceph-mon[24900]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:36:43.371 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:36:43 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:36:43] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:36:43.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:36:43 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:36:43.506Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.106 because it doesn't contain any IP SANs" 2024-04-02T15:36:43.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:36:43 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:36:43.508Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:36:43.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:36:43 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:36:43.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.106 because it doesn't contain any IP SANs" 2024-04-02T15:36:44.031 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:43 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[27385]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:36:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:36:44.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:44 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:44.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:44 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:44.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:44 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:44.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:44 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:44.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:44 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:44.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:44 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:45.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:45 smithi106 ceph-mon[25843]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:45.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:45 smithi087 ceph-mon[29707]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:45.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:45 smithi087 ceph-mon[24900]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:46.010 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:46 smithi106 systemd[1]: Stopping Ceph mgr.x for 0400842c-f105-11ee-b647-cb9ed24678a4... 2024-04-02T15:36:46.367 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:46 smithi106 bash[50481]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr.x" found: no such container 2024-04-02T15:36:46.367 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:46 smithi106 podman[50493]: 2024-04-02 15:36:46.183151655 +0000 UTC m=+0.135264222 container died 7d9b02b111e79df1f5509fac2ab5144ccffc9a5aa8e04b4e580e7087217e460e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x, GIT_CLEAN=True, io.openshift.expose-services=, version=8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, maintainer=Guillaume Abrioux , name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, vendor=Red Hat, Inc., io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, architecture=x86_64) 2024-04-02T15:36:46.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:36:46 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:36:46] "GET /metrics HTTP/1.1" 200 207930 "" "Prometheus/2.33.4" 2024-04-02T15:36:46.803 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:46 smithi106 podman[50493]: 2024-04-02 15:36:46.538187711 +0000 UTC m=+0.490300285 container cleanup 7d9b02b111e79df1f5509fac2ab5144ccffc9a5aa8e04b4e580e7087217e460e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x, architecture=x86_64, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, distribution-scope=public, name=centos-stream, RELEASE=HEAD, com.redhat.component=centos-stream-container, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, GIT_CLEAN=True) 2024-04-02T15:36:46.803 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:46 smithi106 bash[50493]: ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x 2024-04-02T15:36:46.803 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:46 smithi106 bash[50519]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr.x" found: no such container 2024-04-02T15:36:47.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-mon[25843]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:47.117 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:46 smithi106 podman[50507]: 2024-04-02 15:36:46.806178677 +0000 UTC m=+0.616024154 container remove 7d9b02b111e79df1f5509fac2ab5144ccffc9a5aa8e04b4e580e7087217e460e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x, distribution-scope=public, vcs-type=git, release=754, io.openshift.expose-services=, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, io.buildah.version=1.19.8, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, ceph=True, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux ) 2024-04-02T15:36:47.117 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:46 smithi106 systemd[1]: ceph-0400842c-f105-11ee-b647-cb9ed24678a4@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-02T15:36:47.117 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:46 smithi106 systemd[1]: ceph-0400842c-f105-11ee-b647-cb9ed24678a4@mgr.x.service: Failed with result 'exit-code'. 2024-04-02T15:36:47.118 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:46 smithi106 systemd[1]: Stopped Ceph mgr.x for 0400842c-f105-11ee-b647-cb9ed24678a4. 2024-04-02T15:36:47.118 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:46 smithi106 systemd[1]: ceph-0400842c-f105-11ee-b647-cb9ed24678a4@mgr.x.service: Consumed 17.422s CPU time. 2024-04-02T15:36:47.118 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 systemd[1]: Starting Ceph mgr.x for 0400842c-f105-11ee-b647-cb9ed24678a4... 2024-04-02T15:36:47.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:47 smithi087 ceph-mon[29707]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:47.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:47 smithi087 ceph-mon[24900]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:47.372 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:36:47 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:36:47.162Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=5 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:36:47.515 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 podman[50633]: 2024-04-02 15:36:47.139566609 +0000 UTC m=+0.020445364 image pull 7661fe7521b944a40c63d099d9c8c6454b2d4a3af89d0e01e094f668bed96783 quay.ceph.io/ceph-ci/ceph@sha256:6c740d4757bcfc646ba2af58c0d4aaf274efb0f36ed5b987df90c88e7e38aeac 2024-04-02T15:36:47.515 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 podman[50633]: 2024-04-02 15:36:47.257473166 +0000 UTC m=+0.138351901 container create 94f3e590724e5a9967dfe923af1dfe5fbf01c74658694eac3c849aa0f22d0920 (image=quay.ceph.io/ceph-ci/ceph@sha256:6c740d4757bcfc646ba2af58c0d4aaf274efb0f36ed5b987df90c88e7e38aeac, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x, RELEASE=wip-guits-testing-2024-04-02-0703-3f3f3ed, io.buildah.version=1.33.5, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326) 2024-04-02T15:36:47.770 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 podman[50633]: 2024-04-02 15:36:47.518293427 +0000 UTC m=+0.399172176 container init 94f3e590724e5a9967dfe923af1dfe5fbf01c74658694eac3c849aa0f22d0920 (image=quay.ceph.io/ceph-ci/ceph@sha256:6c740d4757bcfc646ba2af58c0d4aaf274efb0f36ed5b987df90c88e7e38aeac, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x, RELEASE=wip-guits-testing-2024-04-02-0703-3f3f3ed, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-02T15:36:47.770 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 podman[50633]: 2024-04-02 15:36:47.521038805 +0000 UTC m=+0.401917543 container start 94f3e590724e5a9967dfe923af1dfe5fbf01c74658694eac3c849aa0f22d0920 (image=quay.ceph.io/ceph-ci/ceph@sha256:6c740d4757bcfc646ba2af58c0d4aaf274efb0f36ed5b987df90c88e7e38aeac, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-2024-04-02-0703-3f3f3ed, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326) 2024-04-02T15:36:47.770 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-mgr[50674]: -- 172.21.15.106:0/150111609 <== mon.2 v2:172.21.15.106:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x561abf69c5a0 con 0x561abf670c00 2024-04-02T15:36:47.770 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.580+0000 7f5751b15200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:36:47.770 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.580+0000 7f5751b15200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:36:47.770 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.770 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.584+0000 7f5751b15200 -1 mgr[py] Class not found in module 'restful' 2024-04-02T15:36:47.770 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.584+0000 7f5751b15200 -1 mgr[py] Error loading module 'restful': (22) Invalid argument 2024-04-02T15:36:47.770 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.599+0000 7f5751b15200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:36:47.770 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.599+0000 7f5751b15200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:36:47.771 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.771 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.603+0000 7f5751b15200 -1 mgr[py] Class not found in module 'mds_autoscaler' 2024-04-02T15:36:47.771 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.603+0000 7f5751b15200 -1 mgr[py] Error loading module 'mds_autoscaler': (22) Invalid argument 2024-04-02T15:36:47.771 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 bash[50633]: 94f3e590724e5a9967dfe923af1dfe5fbf01c74658694eac3c849aa0f22d0920 2024-04-02T15:36:47.771 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.618+0000 7f5751b15200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:36:47.771 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 systemd[1]: Started Ceph mgr.x for 0400842c-f105-11ee-b647-cb9ed24678a4. 2024-04-02T15:36:47.771 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.618+0000 7f5751b15200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:36:47.771 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.771 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.624+0000 7f5751b15200 -1 mgr[py] Class not found in module 'osd_perf_query' 2024-04-02T15:36:47.772 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.624+0000 7f5751b15200 -1 mgr[py] Error loading module 'osd_perf_query': (22) Invalid argument 2024-04-02T15:36:47.772 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.642+0000 7f5751b15200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:36:47.772 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.642+0000 7f5751b15200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:36:47.772 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.772 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.648+0000 7f5751b15200 -1 mgr[py] Class not found in module 'snap_schedule' 2024-04-02T15:36:47.772 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.648+0000 7f5751b15200 -1 mgr[py] Error loading module 'snap_schedule': (22) Invalid argument 2024-04-02T15:36:47.772 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.665+0000 7f5751b15200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:36:47.772 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.665+0000 7f5751b15200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:36:47.772 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.772 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.669+0000 7f5751b15200 -1 mgr[py] Class not found in module 'dashboard' 2024-04-02T15:36:47.772 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.669+0000 7f5751b15200 -1 mgr[py] Error loading module 'dashboard': (22) Invalid argument 2024-04-02T15:36:47.772 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.686+0000 7f5751b15200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:36:47.772 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.686+0000 7f5751b15200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:36:47.772 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.773 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.692+0000 7f5751b15200 -1 mgr[py] Class not found in module 'zabbix' 2024-04-02T15:36:47.773 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.692+0000 7f5751b15200 -1 mgr[py] Error loading module 'zabbix': (22) Invalid argument 2024-04-02T15:36:47.773 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.710+0000 7f5751b15200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:36:47.773 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.710+0000 7f5751b15200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:36:47.773 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.773 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.715+0000 7f5751b15200 -1 mgr[py] Class not found in module 'balancer' 2024-04-02T15:36:47.773 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.715+0000 7f5751b15200 -1 mgr[py] Error loading module 'balancer': (22) Invalid argument 2024-04-02T15:36:47.773 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.734+0000 7f5751b15200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:36:47.773 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.734+0000 7f5751b15200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:36:47.773 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.773 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.738+0000 7f5751b15200 -1 mgr[py] Class not found in module 'influx' 2024-04-02T15:36:47.773 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.738+0000 7f5751b15200 -1 mgr[py] Error loading module 'influx': (22) Invalid argument 2024-04-02T15:36:47.773 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.767+0000 7f5751b15200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:36:47.773 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.767+0000 7f5751b15200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:36:47.773 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.773 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.771+0000 7f5751b15200 -1 mgr[py] Class not found in module 'alerts' 2024-04-02T15:36:47.774 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.771+0000 7f5751b15200 -1 mgr[py] Error loading module 'alerts': (22) Invalid argument 2024-04-02T15:36:48.020 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.788+0000 7f5751b15200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:36:48.021 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.788+0000 7f5751b15200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:36:48.021 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.021 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.792+0000 7f5751b15200 -1 mgr[py] Class not found in module 'iostat' 2024-04-02T15:36:48.021 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.792+0000 7f5751b15200 -1 mgr[py] Error loading module 'iostat': (22) Invalid argument 2024-04-02T15:36:48.021 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.807+0000 7f5751b15200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:36:48.021 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.807+0000 7f5751b15200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:36:48.021 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.021 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.812+0000 7f5751b15200 -1 mgr[py] Class not found in module 'rgw' 2024-04-02T15:36:48.021 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.812+0000 7f5751b15200 -1 mgr[py] Error loading module 'rgw': (22) Invalid argument 2024-04-02T15:36:48.021 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.829+0000 7f5751b15200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:36:48.021 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.829+0000 7f5751b15200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:36:48.021 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.021 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.833+0000 7f5751b15200 -1 mgr[py] Class not found in module 'rbd_support' 2024-04-02T15:36:48.021 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.833+0000 7f5751b15200 -1 mgr[py] Error loading module 'rbd_support': (22) Invalid argument 2024-04-02T15:36:48.022 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.848+0000 7f5751b15200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:36:48.022 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.848+0000 7f5751b15200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:36:48.022 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.022 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.852+0000 7f5751b15200 -1 mgr[py] Class not found in module 'progress' 2024-04-02T15:36:48.022 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.852+0000 7f5751b15200 -1 mgr[py] Error loading module 'progress': (22) Invalid argument 2024-04-02T15:36:48.022 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.867+0000 7f5751b15200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:36:48.022 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.867+0000 7f5751b15200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:36:48.022 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.022 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.870+0000 7f5751b15200 -1 mgr[py] Class not found in module 'pg_autoscaler' 2024-04-02T15:36:48.022 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.870+0000 7f5751b15200 -1 mgr[py] Error loading module 'pg_autoscaler': (22) Invalid argument 2024-04-02T15:36:48.023 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.885+0000 7f5751b15200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:36:48.023 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.885+0000 7f5751b15200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:36:48.023 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.023 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.889+0000 7f5751b15200 -1 mgr[py] Class not found in module 'devicehealth' 2024-04-02T15:36:48.023 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.889+0000 7f5751b15200 -1 mgr[py] Error loading module 'devicehealth': (22) Invalid argument 2024-04-02T15:36:48.023 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.904+0000 7f5751b15200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:36:48.023 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.904+0000 7f5751b15200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:36:48.023 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.023 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.908+0000 7f5751b15200 -1 mgr[py] Class not found in module 'k8sevents' 2024-04-02T15:36:48.023 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.908+0000 7f5751b15200 -1 mgr[py] Error loading module 'k8sevents': (22) Invalid argument 2024-04-02T15:36:48.023 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.923+0000 7f5751b15200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:36:48.023 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.923+0000 7f5751b15200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:36:48.024 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.024 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.928+0000 7f5751b15200 -1 mgr[py] Class not found in module 'rook' 2024-04-02T15:36:48.024 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.928+0000 7f5751b15200 -1 mgr[py] Error loading module 'rook': (22) Invalid argument 2024-04-02T15:36:48.024 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.943+0000 7f5751b15200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:36:48.024 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.943+0000 7f5751b15200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:36:48.024 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.024 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.947+0000 7f5751b15200 -1 mgr[py] Class not found in module 'diskprediction_local' 2024-04-02T15:36:48.024 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.947+0000 7f5751b15200 -1 mgr[py] Error loading module 'diskprediction_local': (22) Invalid argument 2024-04-02T15:36:48.024 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.985+0000 7f5751b15200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:36:48.024 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.985+0000 7f5751b15200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:36:48.024 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.024 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.989+0000 7f5751b15200 -1 mgr[py] Class not found in module 'mirroring' 2024-04-02T15:36:48.024 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:47 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:47.989+0000 7f5751b15200 -1 mgr[py] Error loading module 'mirroring': (22) Invalid argument 2024-04-02T15:36:48.024 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.003+0000 7f5751b15200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:36:48.024 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.003+0000 7f5751b15200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:36:48.024 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.024 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.007+0000 7f5751b15200 -1 mgr[py] Class not found in module 'selftest' 2024-04-02T15:36:48.025 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.007+0000 7f5751b15200 -1 mgr[py] Error loading module 'selftest': (22) Invalid argument 2024-04-02T15:36:48.025 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.022+0000 7f5751b15200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:36:48.278 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.022+0000 7f5751b15200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:36:48.278 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.278 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.026+0000 7f5751b15200 -1 mgr[py] Class not found in module 'telegraf' 2024-04-02T15:36:48.278 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.026+0000 7f5751b15200 -1 mgr[py] Error loading module 'telegraf': (22) Invalid argument 2024-04-02T15:36:48.278 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.041+0000 7f5751b15200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:36:48.278 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.041+0000 7f5751b15200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:36:48.278 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.279 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.045+0000 7f5751b15200 -1 mgr[py] Class not found in module 'test_orchestrator' 2024-04-02T15:36:48.279 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.045+0000 7f5751b15200 -1 mgr[py] Error loading module 'test_orchestrator': (22) Invalid argument 2024-04-02T15:36:48.279 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.061+0000 7f5751b15200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:36:48.279 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.061+0000 7f5751b15200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:36:48.279 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.279 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.065+0000 7f5751b15200 -1 mgr[py] Class not found in module 'localpool' 2024-04-02T15:36:48.279 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.065+0000 7f5751b15200 -1 mgr[py] Error loading module 'localpool': (22) Invalid argument 2024-04-02T15:36:48.279 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.080+0000 7f5751b15200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:36:48.279 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.080+0000 7f5751b15200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:36:48.279 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.279 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.085+0000 7f5751b15200 -1 mgr[py] Class not found in module 'crash' 2024-04-02T15:36:48.279 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.085+0000 7f5751b15200 -1 mgr[py] Error loading module 'crash': (22) Invalid argument 2024-04-02T15:36:48.279 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.100+0000 7f5751b15200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:36:48.279 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.100+0000 7f5751b15200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:36:48.279 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.280 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.105+0000 7f5751b15200 -1 mgr[py] Class not found in module 'insights' 2024-04-02T15:36:48.280 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.105+0000 7f5751b15200 -1 mgr[py] Error loading module 'insights': (22) Invalid argument 2024-04-02T15:36:48.280 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.120+0000 7f5751b15200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:36:48.280 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.120+0000 7f5751b15200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:36:48.280 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.280 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.124+0000 7f5751b15200 -1 mgr[py] Class not found in module 'orchestrator' 2024-04-02T15:36:48.280 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.124+0000 7f5751b15200 -1 mgr[py] Error loading module 'orchestrator': (22) Invalid argument 2024-04-02T15:36:48.280 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.139+0000 7f5751b15200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:36:48.280 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.139+0000 7f5751b15200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:36:48.280 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.281 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.143+0000 7f5751b15200 -1 mgr[py] Class not found in module 'osd_support' 2024-04-02T15:36:48.281 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.143+0000 7f5751b15200 -1 mgr[py] Error loading module 'osd_support': (22) Invalid argument 2024-04-02T15:36:48.281 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.158+0000 7f5751b15200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:36:48.281 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.158+0000 7f5751b15200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:36:48.281 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.281 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.162+0000 7f5751b15200 -1 mgr[py] Class not found in module 'volumes' 2024-04-02T15:36:48.281 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.162+0000 7f5751b15200 -1 mgr[py] Error loading module 'volumes': (22) Invalid argument 2024-04-02T15:36:48.281 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.217+0000 7f5751b15200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:36:48.281 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.217+0000 7f5751b15200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:36:48.281 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.281 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.223+0000 7f5751b15200 -1 mgr[py] Class not found in module 'stats' 2024-04-02T15:36:48.281 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.223+0000 7f5751b15200 -1 mgr[py] Error loading module 'stats': (22) Invalid argument 2024-04-02T15:36:48.281 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.239+0000 7f5751b15200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:36:48.281 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.239+0000 7f5751b15200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:36:48.281 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.282 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.244+0000 7f5751b15200 -1 mgr[py] Class not found in module 'cephadm' 2024-04-02T15:36:48.282 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.244+0000 7f5751b15200 -1 mgr[py] Error loading module 'cephadm': (22) Invalid argument 2024-04-02T15:36:48.282 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.260+0000 7f5751b15200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:36:48.282 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.260+0000 7f5751b15200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:36:48.282 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.282 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.264+0000 7f5751b15200 -1 mgr[py] Class not found in module 'telemetry' 2024-04-02T15:36:48.282 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.264+0000 7f5751b15200 -1 mgr[py] Error loading module 'telemetry': (22) Invalid argument 2024-04-02T15:36:48.282 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.279+0000 7f5751b15200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:36:48.617 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.279+0000 7f5751b15200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:36:48.618 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.618 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.285+0000 7f5751b15200 -1 mgr[py] Class not found in module 'prometheus' 2024-04-02T15:36:48.618 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.285+0000 7f5751b15200 -1 mgr[py] Error loading module 'prometheus': (22) Invalid argument 2024-04-02T15:36:48.618 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.301+0000 7f5751b15200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:36:48.618 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.301+0000 7f5751b15200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:36:48.618 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.618 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.305+0000 7f5751b15200 -1 mgr[py] Class not found in module 'status' 2024-04-02T15:36:48.618 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.305+0000 7f5751b15200 -1 mgr[py] Error loading module 'status': (22) Invalid argument 2024-04-02T15:36:48.618 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.321+0000 7f5751b15200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-02T15:36:48.618 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.321+0000 7f5751b15200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-02T15:36:48.618 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.618 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.326+0000 7f5751b15200 -1 mgr[py] Class not found in module 'nfs' 2024-04-02T15:36:48.618 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.326+0000 7f5751b15200 -1 mgr[py] Error loading module 'nfs': (22) Invalid argument 2024-04-02T15:36:48.618 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:36:48.326+0000 7f5751b15200 -1 log_channel(cluster) log [ERR] : Failed to load ceph-mgr modules: restful, mds_autoscaler, osd_perf_query, snap_schedule, dashboard, zabbix, balancer, influx, alerts, iostat, rgw, rbd_support, progress, pg_autoscaler, devicehealth, k8sevents, rook, diskprediction_local, mirroring, selftest, telegraf, test_orchestrator, localpool, crash, insights, orchestrator, osd_support, volumes, stats, cephadm, telemetry, prometheus, status, nfs 2024-04-02T15:36:48.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:48 smithi087 ceph-mon[29707]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:36:48.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:48 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:48.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:48 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:48.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:48 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:36:48.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:48 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:36:48.872 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:48 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:36:48.872 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:48 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:48.872 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:48 smithi087 ceph-mon[29707]: Standby manager daemon x restarted 2024-04-02T15:36:48.872 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:48 smithi087 ceph-mon[29707]: Standby manager daemon x started 2024-04-02T15:36:48.872 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:48 smithi087 ceph-mon[24900]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:36:48.872 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:48 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:48.872 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:48 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:48.872 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:48 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:36:48.872 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:48 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:36:48.872 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:48 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:36:48.872 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:48 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:48.872 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:48 smithi087 ceph-mon[24900]: Standby manager daemon x restarted 2024-04-02T15:36:48.873 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:48 smithi087 ceph-mon[24900]: Standby manager daemon x started 2024-04-02T15:36:49.026 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-mon[25843]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:36:49.026 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:49.026 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:49.026 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:36:49.026 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:36:49.026 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:36:49.026 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:49.026 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-mon[25843]: Standby manager daemon x restarted 2024-04-02T15:36:49.026 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:48 smithi106 ceph-mon[25843]: Standby manager daemon x started 2024-04-02T15:36:50.214 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:50 smithi106 ceph-mon[25843]: mgrmap e21: y(active, since 7m), standbys: x 2024-04-02T15:36:50.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:50 smithi087 ceph-mon[24900]: mgrmap e21: y(active, since 7m), standbys: x 2024-04-02T15:36:50.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:50 smithi087 ceph-mon[29707]: mgrmap e21: y(active, since 7m), standbys: x 2024-04-02T15:36:51.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:51 smithi106 ceph-mon[25843]: Failed to load ceph-mgr modules: restful, mds_autoscaler, osd_perf_query, snap_schedule, dashboard, zabbix, balancer, influx, alerts, iostat, rgw, rbd_support, progress, pg_autoscaler, devicehealth, k8sevents, rook, diskprediction_local, mirroring, selftest, telegraf, test_orchestrator, localpool, crash, insights, orchestrator, osd_support, volumes, stats, cephadm, telemetry, prometheus, status, nfs 2024-04-02T15:36:51.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:51 smithi106 ceph-mon[25843]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:51.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:51 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:36:51.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:51 smithi087 ceph-mon[29707]: Failed to load ceph-mgr modules: restful, mds_autoscaler, osd_perf_query, snap_schedule, dashboard, zabbix, balancer, influx, alerts, iostat, rgw, rbd_support, progress, pg_autoscaler, devicehealth, k8sevents, rook, diskprediction_local, mirroring, selftest, telegraf, test_orchestrator, localpool, crash, insights, orchestrator, osd_support, volumes, stats, cephadm, telemetry, prometheus, status, nfs 2024-04-02T15:36:51.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:51 smithi087 ceph-mon[29707]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:51.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:51 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:36:51.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:51 smithi087 ceph-mon[24900]: Failed to load ceph-mgr modules: restful, mds_autoscaler, osd_perf_query, snap_schedule, dashboard, zabbix, balancer, influx, alerts, iostat, rgw, rbd_support, progress, pg_autoscaler, devicehealth, k8sevents, rook, diskprediction_local, mirroring, selftest, telegraf, test_orchestrator, localpool, crash, insights, orchestrator, osd_support, volumes, stats, cephadm, telemetry, prometheus, status, nfs 2024-04-02T15:36:51.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:51 smithi087 ceph-mon[24900]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:51.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:51 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:36:53.051 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:36:52 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:36:52] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:36:53.051 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:36:52 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:36:52] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:36:53.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:53 smithi106 ceph-mon[25843]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:36:53.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:53 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:53.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:53 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:53.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:53 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:53.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:53 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:53.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:53 smithi087 ceph-mon[29707]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:36:53.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:53 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:53.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:53 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:53.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:53 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:53.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:53 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:53.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:53 smithi087 ceph-mon[24900]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:36:53.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:53 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:53.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:53 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:53.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:53 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:53.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:53 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:36:53.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:36:53 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:36:53.506Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:36:53.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:36:53 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:36:53.507Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:36:53.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:36:53 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:36:53.508Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:36:55.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:55 smithi106 ceph-mon[25843]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:55.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:55 smithi087 ceph-mon[29707]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:55.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:55 smithi087 ceph-mon[24900]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:56.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:36:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:36:56] "GET /metrics HTTP/1.1" 200 207934 "" "Prometheus/2.33.4" 2024-04-02T15:36:57.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:57 smithi106 ceph-mon[25843]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:57.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:57 smithi087 ceph-mon[29707]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:57.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:57 smithi087 ceph-mon[24900]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:36:59.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:36:59 smithi106 ceph-mon[25843]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:36:59.365 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:36:59 smithi087 ceph-mon[29707]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:36:59.365 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:36:59 smithi087 ceph-mon[24900]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:37:00.558 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:00 smithi106 ceph-mon[25843]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:00.559 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:00 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:37:00.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:00 smithi087 ceph-mon[29707]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:00.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:00 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:37:00.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:00 smithi087 ceph-mon[24900]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:00.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:00 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:37:02.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:02 smithi106 ceph-mon[25843]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:37:02.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:02 smithi087 ceph-mon[24900]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:37:02.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:02 smithi087 ceph-mon[29707]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:37:03.505 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:37:03 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:37:03] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:37:03.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:37:03 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:37:03.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:37:03.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:37:03 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:37:03.508Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:37:03.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:37:03 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:37:03.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:37:04.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:04 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:37:04.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:04 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:37:04.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:04 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:37:04.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:04 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:37:04.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:04 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:37:04.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:04 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:37:05.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:05 smithi106 ceph-mon[25843]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:05.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:05 smithi087 ceph-mon[29707]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:05.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:05 smithi087 ceph-mon[24900]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:06.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:37:06 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:37:06] "GET /metrics HTTP/1.1" 200 207935 "" "Prometheus/2.33.4" 2024-04-02T15:37:07.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:07 smithi106 ceph-mon[25843]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:07.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:07 smithi087 ceph-mon[29707]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:07.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:07 smithi087 ceph-mon[24900]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:09.364 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:09 smithi087 ceph-mon[29707]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:37:09.364 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:09 smithi087 ceph-mon[24900]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:37:09.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:09 smithi106 ceph-mon[25843]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:37:10.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:10 smithi087 ceph-mon[29707]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:10.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:10 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:37:10.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:10 smithi087 ceph-mon[24900]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:10.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:10 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:37:10.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:10 smithi106 ceph-mon[25843]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:10.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:10 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:37:12.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:12 smithi106 ceph-mon[25843]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:37:12.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:12 smithi087 ceph-mon[29707]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:37:12.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:12 smithi087 ceph-mon[24900]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:37:13.371 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:37:12 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:37:12] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:37:13.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:37:13 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:37:13.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:37:13.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:37:13 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:37:13.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:37:13.873 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:37:13 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:37:13.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:37:14.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:13 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:37:14.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:13 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:37:14.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:13 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:37:14.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:13 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:37:14.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:13 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:37:14.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:13 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:37:15.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:14 smithi106 ceph-mon[25843]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:15.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:14 smithi087 ceph-mon[29707]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:15.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:14 smithi087 ceph-mon[24900]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:16.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:37:16 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:37:16] "GET /metrics HTTP/1.1" 200 207935 "" "Prometheus/2.33.4" 2024-04-02T15:37:17.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:16 smithi106 ceph-mon[25843]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:17.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:16 smithi087 ceph-mon[29707]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:17.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:16 smithi087 ceph-mon[24900]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:19.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:18 smithi106 ceph-mon[25843]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:37:19.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:18 smithi087 ceph-mon[24900]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:37:19.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:18 smithi087 ceph-mon[29707]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:37:20.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:20 smithi106 ceph-mon[25843]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:20.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:20 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:37:20.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:20 smithi087 ceph-mon[29707]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:20.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:20 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:37:20.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:20 smithi087 ceph-mon[24900]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:20.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:20 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:37:21.371 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:37:20 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:37:20] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:37:22.117 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:21 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:37:22.118 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:21 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:37:22.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:21 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:37:22.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:21 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:37:22.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:21 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:37:22.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:21 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:37:23.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:22 smithi106 ceph-mon[25843]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:37:23.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:22 smithi087 ceph-mon[29707]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:37:23.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:22 smithi087 ceph-mon[24900]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:37:23.372 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:37:23 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:37:23] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:37:23.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:37:23 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:37:23.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:37:23.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:37:23 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:37:23.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:37:23.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:37:23 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:37:23.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:37:24.573 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:24 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:37:24.573 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:24 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:37:24.573 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:24 smithi087 ceph-mon[29707]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:24.573 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:24 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:37:24.573 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:24 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:37:24.573 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:24 smithi087 ceph-mon[24900]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:24.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:24 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:37:24.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:24 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:37:24.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:24 smithi106 ceph-mon[25843]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:26.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:26 smithi087 ceph-mon[29707]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:26.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:26 smithi087 ceph-mon[24900]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:26.622 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:37:26 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:37:26] "GET /metrics HTTP/1.1" 200 207933 "" "Prometheus/2.33.4" 2024-04-02T15:37:26.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:26 smithi106 ceph-mon[25843]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:28.713 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:28 smithi106 ceph-mon[25843]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:37:28.867 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:28 smithi087 ceph-mon[29707]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:37:28.867 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:28 smithi087 ceph-mon[24900]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:37:30.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:30 smithi106 ceph-mon[25843]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:30.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:30 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:37:30.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:30 smithi087 ceph-mon[24900]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:30.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:30 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:37:30.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:30 smithi087 ceph-mon[29707]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:30.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:30 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:37:32.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:32 smithi106 ceph-mon[25843]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:37:32.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:32 smithi087 ceph-mon[29707]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:37:32.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:32 smithi087 ceph-mon[24900]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:37:33.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:37:33 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:37:33.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:37:33.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:37:33 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:37:33.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:37:33.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:37:33 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:37:33.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:37:34.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:34 smithi106 ceph-mon[25843]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:34.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:34 smithi087 ceph-mon[24900]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:34.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:34 smithi087 ceph-mon[29707]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:35.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:35 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:37:35.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:35 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:37:35.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:35 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:37:35.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:35 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:37:35.872 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:35 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-02T15:37:35.872 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:35 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-02T15:37:36.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:36 smithi087 ceph-mon[29707]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:36.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:36 smithi087 ceph-mon[24900]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:36.622 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:37:36 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:37:36] "GET /metrics HTTP/1.1" 200 207934 "" "Prometheus/2.33.4" 2024-04-02T15:37:36.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:36 smithi106 ceph-mon[25843]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:38.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:38 smithi106 ceph-mon[25843]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:37:38.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:38 smithi087 ceph-mon[29707]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:37:38.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:38 smithi087 ceph-mon[24900]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:37:40.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:40 smithi106 ceph-mon[25843]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:40.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:40 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:37:40.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:40 smithi087 ceph-mon[29707]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:40.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:40 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:37:40.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:40 smithi087 ceph-mon[24900]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:40.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:40 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:37:41.371 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:37:41 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:37:41] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:37:42.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:42 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:37:42.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:42 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:37:42.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:42 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:37:42.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:42 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:37:42.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:42 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:37:42.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:42 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:37:43.367 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:43 smithi106 ceph-mon[25843]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:37:43.371 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:43 smithi087 ceph-mon[24900]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:37:43.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:43 smithi087 ceph-mon[29707]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:37:43.372 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:37:43 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:37:43] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:37:43.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:37:43 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:37:43.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:37:43.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:37:43 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:37:43.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:37:43.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:37:43 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:37:43.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:37:44.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:44 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:37:44.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:44 smithi106 ceph-mon[25843]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:44.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:44 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:37:44.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:44 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:37:44.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:44 smithi087 ceph-mon[24900]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:44.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:44 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:37:44.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:44 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:37:44.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:44 smithi087 ceph-mon[29707]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:44.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:44 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:37:46.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:46 smithi087 ceph-mon[24900]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:46.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:46 smithi087 ceph-mon[29707]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:46.622 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:37:46 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:37:46] "GET /metrics HTTP/1.1" 200 207934 "" "Prometheus/2.33.4" 2024-04-02T15:37:46.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:46 smithi106 ceph-mon[25843]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:48.508 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:48 smithi106 ceph-mon[25843]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:37:48.508 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:48 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:37:48.508 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:48 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:37:48.508 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:48 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:37:48.508 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:48 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:37:48.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:48 smithi087 ceph-mon[29707]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:37:48.872 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:48 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:37:48.872 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:48 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:37:48.872 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:48 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:37:48.872 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:48 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:37:48.872 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:48 smithi087 ceph-mon[24900]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:37:48.872 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:48 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T15:37:48.872 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:48 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T15:37:48.872 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:48 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T15:37:48.872 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:48 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:37:50.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:50 smithi106 ceph-mon[25843]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:50.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:50 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:37:50.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:50 smithi087 ceph-mon[24900]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:50.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:50 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:37:50.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:50 smithi087 ceph-mon[29707]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:50.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:50 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:37:52.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:52 smithi087 ceph-mon[24900]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:37:52.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:52 smithi087 ceph-mon[29707]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:37:52.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:52 smithi106 ceph-mon[25843]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:37:53.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:37:53 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:37:53.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:37:53.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:37:53 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:37:53.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:37:53.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:37:53 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:37:53.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:37:54.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:54 smithi106 ceph-mon[25843]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:54.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:54 smithi087 ceph-mon[29707]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:54.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:54 smithi087 ceph-mon[24900]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:56.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:56 smithi087 ceph-mon[29707]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:56.622 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:37:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:37:56] "GET /metrics HTTP/1.1" 200 207942 "" "Prometheus/2.33.4" 2024-04-02T15:37:56.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:56 smithi087 ceph-mon[24900]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:56.669 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:56 smithi106 ceph-mon[25843]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:37:58.811 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:37:58 smithi106 ceph-mon[25843]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:37:58.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:37:58 smithi087 ceph-mon[29707]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:37:58.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:37:58 smithi087 ceph-mon[24900]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:38:00.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:00 smithi106 ceph-mon[25843]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:38:00.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:00 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:38:00.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:00 smithi087 ceph-mon[29707]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:38:00.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:00 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:38:00.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:00 smithi087 ceph-mon[24900]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:38:00.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:00 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:38:01.448 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:01 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:38:01] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:38:02.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:02 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:38:02.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:02 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:38:02.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:02 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:38:02.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:02 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:38:02.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:02 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:38:02.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:02 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:38:03.510 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:03 smithi087 ceph-mon[29707]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:38:03.510 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:03 smithi087 ceph-mon[24900]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:38:03.510 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:03 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.87 - - [02/Apr/2024:15:38:03] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:38:03.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:03 smithi106 ceph-mon[25843]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:38:03.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:38:03 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:38:03.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:38:03.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:38:03 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:38:03.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:38:03.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:38:03 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:38:03.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:38:04.615 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:04 smithi106 ceph-mon[25843]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:38:04.615 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:04 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:38:04.615 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:04 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:38:04.734 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:04 smithi087 ceph-mon[29707]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:38:04.734 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:04 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:38:04.735 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:04 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:38:04.735 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:04 smithi087 ceph-mon[24900]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:38:04.735 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:04 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:38:04.735 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:04 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:38:05.172 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph orch ps' 2024-04-02T15:38:06.522 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:06 smithi106 ceph-mon[25843]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:38:06.620 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:06 smithi087 ceph-mon[29707]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:38:06.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:06 smithi087 ceph-mon[24900]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:38:06.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:06 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:38:06] "GET /metrics HTTP/1.1" 200 207943 "" "Prometheus/2.33.4" 2024-04-02T15:38:07.161 INFO:teuthology.orchestra.run.smithi087.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T15:38:07.162 INFO:teuthology.orchestra.run.smithi087.stdout:agent.smithi087 smithi087 running 3s ago 12m - - 2024-04-02T15:38:07.162 INFO:teuthology.orchestra.run.smithi087.stdout:agent.smithi106 smithi106 running 5s ago 12m - - 2024-04-02T15:38:07.162 INFO:teuthology.orchestra.run.smithi087.stdout:alertmanager.a smithi087 *:9093,9094 running (6m) 3s ago 7m 27.4M - ba2b418f427c 93850beda829 2024-04-02T15:38:07.162 INFO:teuthology.orchestra.run.smithi087.stdout:grafana.a smithi106 *:3000 running (6m) 5s ago 6m 55.8M - 8.3.5 dad864ee21e9 8956d795b0b4 2024-04-02T15:38:07.162 INFO:teuthology.orchestra.run.smithi087.stdout:iscsi.foo.smithi087.icnqbx smithi087 running (6m) 3s ago 6m 56.7M - 3.5 e1d6a67b021e 8278ac6ea2a5 2024-04-02T15:38:07.162 INFO:teuthology.orchestra.run.smithi087.stdout:mgr.x smithi106 *:8443,9283 running (80s) 5s ago 11m 101M - 19.0.0-2592-g3f3f3ed9 7661fe7521b9 94f3e590724e 2024-04-02T15:38:07.162 INFO:teuthology.orchestra.run.smithi087.stdout:mgr.y smithi087 *:9283 running (13m) 3s ago 13m 469M - 17.2.0 e1d6a67b021e 5d422ff3cc7d 2024-04-02T15:38:07.162 INFO:teuthology.orchestra.run.smithi087.stdout:mon.a smithi087 running (13m) 3s ago 13m 52.5M 2048M 17.2.0 e1d6a67b021e e28fd32a7d9a 2024-04-02T15:38:07.162 INFO:teuthology.orchestra.run.smithi087.stdout:mon.b smithi106 running (12m) 5s ago 12m 42.4M 2048M 17.2.0 e1d6a67b021e d9de5ac6a73e 2024-04-02T15:38:07.162 INFO:teuthology.orchestra.run.smithi087.stdout:mon.c smithi087 running (12m) 3s ago 12m 44.4M 2048M 17.2.0 e1d6a67b021e 40f23934618c 2024-04-02T15:38:07.162 INFO:teuthology.orchestra.run.smithi087.stdout:node-exporter.a smithi087 *:9100 running (8m) 3s ago 8m 18.1M - 1dbe0e931976 feba1ed50125 2024-04-02T15:38:07.162 INFO:teuthology.orchestra.run.smithi087.stdout:node-exporter.b smithi106 *:9100 running (8m) 5s ago 8m 15.4M - 1dbe0e931976 fb10fb2b50d2 2024-04-02T15:38:07.162 INFO:teuthology.orchestra.run.smithi087.stdout:osd.0 smithi087 running (11m) 3s ago 11m 51.0M 4096M 17.2.0 e1d6a67b021e 644fbe6ff2bc 2024-04-02T15:38:07.163 INFO:teuthology.orchestra.run.smithi087.stdout:osd.1 smithi087 running (11m) 3s ago 11m 50.7M 4096M 17.2.0 e1d6a67b021e 8996e84013c2 2024-04-02T15:38:07.163 INFO:teuthology.orchestra.run.smithi087.stdout:osd.2 smithi087 running (10m) 3s ago 10m 48.7M 4096M 17.2.0 e1d6a67b021e 944a519c2f4b 2024-04-02T15:38:07.163 INFO:teuthology.orchestra.run.smithi087.stdout:osd.3 smithi087 running (10m) 3s ago 10m 48.3M 4096M 17.2.0 e1d6a67b021e 00c03ac366b2 2024-04-02T15:38:07.163 INFO:teuthology.orchestra.run.smithi087.stdout:osd.4 smithi106 running (10m) 5s ago 10m 50.2M 4096M 17.2.0 e1d6a67b021e c96f419b11f6 2024-04-02T15:38:07.163 INFO:teuthology.orchestra.run.smithi087.stdout:osd.5 smithi106 running (9m) 5s ago 9m 50.2M 4096M 17.2.0 e1d6a67b021e 91062a17779c 2024-04-02T15:38:07.163 INFO:teuthology.orchestra.run.smithi087.stdout:osd.6 smithi106 running (9m) 5s ago 9m 49.1M 4096M 17.2.0 e1d6a67b021e f582842222f2 2024-04-02T15:38:07.163 INFO:teuthology.orchestra.run.smithi087.stdout:osd.7 smithi106 running (8m) 5s ago 8m 49.7M 4096M 17.2.0 e1d6a67b021e 85ed81d59bdd 2024-04-02T15:38:07.163 INFO:teuthology.orchestra.run.smithi087.stdout:prometheus.a smithi106 *:9095 running (6m) 5s ago 8m 67.2M - 514e6a882f6e 3ea0ba4743fb 2024-04-02T15:38:07.163 INFO:teuthology.orchestra.run.smithi087.stdout:rgw.foo.smithi087.awfxzs smithi087 *:8000 running (6m) 3s ago 6m 72.9M - 17.2.0 e1d6a67b021e ecaa755cb095 2024-04-02T15:38:07.163 INFO:teuthology.orchestra.run.smithi087.stdout:rgw.foo.smithi106.bfeikp smithi106 *:8000 running (6m) 5s ago 6m 72.8M - 17.2.0 e1d6a67b021e a6dc1247c030 2024-04-02T15:38:07.849 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:07 smithi087 ceph-mon[29707]: from='client.15018 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:38:07.849 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:07 smithi087 ceph-mon[24900]: from='client.15018 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:38:07.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:07 smithi106 ceph-mon[25843]: from='client.15018 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T15:38:07.875 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph versions' 2024-04-02T15:38:08.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:08 smithi106 ceph-mon[25843]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:38:08.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:08 smithi087 ceph-mon[29707]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:38:08.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:08 smithi087 ceph-mon[24900]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:38:09.992 INFO:teuthology.orchestra.run.smithi087.stdout:{ 2024-04-02T15:38:09.992 INFO:teuthology.orchestra.run.smithi087.stdout: "mon": { 2024-04-02T15:38:09.992 INFO:teuthology.orchestra.run.smithi087.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-02T15:38:09.992 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-02T15:38:09.993 INFO:teuthology.orchestra.run.smithi087.stdout: "mgr": { 2024-04-02T15:38:09.993 INFO:teuthology.orchestra.run.smithi087.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-02T15:38:09.993 INFO:teuthology.orchestra.run.smithi087.stdout: "ceph version 19.0.0-2592-g3f3f3ed9 (3f3f3ed992429b85be729743ad3b39d52dd90744) squid (dev)": 1 2024-04-02T15:38:09.993 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-02T15:38:09.993 INFO:teuthology.orchestra.run.smithi087.stdout: "osd": { 2024-04-02T15:38:09.993 INFO:teuthology.orchestra.run.smithi087.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-02T15:38:09.993 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-02T15:38:09.993 INFO:teuthology.orchestra.run.smithi087.stdout: "mds": {}, 2024-04-02T15:38:09.993 INFO:teuthology.orchestra.run.smithi087.stdout: "rgw": { 2024-04-02T15:38:09.993 INFO:teuthology.orchestra.run.smithi087.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-02T15:38:09.993 INFO:teuthology.orchestra.run.smithi087.stdout: }, 2024-04-02T15:38:09.993 INFO:teuthology.orchestra.run.smithi087.stdout: "overall": { 2024-04-02T15:38:09.993 INFO:teuthology.orchestra.run.smithi087.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14, 2024-04-02T15:38:09.993 INFO:teuthology.orchestra.run.smithi087.stdout: "ceph version 19.0.0-2592-g3f3f3ed9 (3f3f3ed992429b85be729743ad3b39d52dd90744) squid (dev)": 1 2024-04-02T15:38:09.993 INFO:teuthology.orchestra.run.smithi087.stdout: } 2024-04-02T15:38:09.993 INFO:teuthology.orchestra.run.smithi087.stdout:} 2024-04-02T15:38:10.606 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:10 smithi087 ceph-mon[29707]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:38:10.606 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:10 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:38:10.606 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:10 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/25768004' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T15:38:10.606 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:10 smithi087 ceph-mon[24900]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:38:10.606 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:10 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:38:10.607 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:10 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/25768004' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T15:38:10.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:10 smithi106 ceph-mon[25843]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:38:10.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:10 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:38:10.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:10 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/25768004' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T15:38:10.632 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph -s' 2024-04-02T15:38:11.664 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:38:11 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:38:11.626Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:38:11.664 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:38:11 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:38:11.627Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:38:12.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:12 smithi087 ceph-mon[29707]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:38:12.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:12 smithi087 ceph-mon[24900]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:38:12.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:12 smithi106 ceph-mon[25843]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:38:13.621 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:38:13 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:38:13.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:38:13.622 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:38:13 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:38:13.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:38:13.622 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:38:13 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:38:13.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:38:13.706 INFO:teuthology.orchestra.run.smithi087.stdout: cluster: 2024-04-02T15:38:13.706 INFO:teuthology.orchestra.run.smithi087.stdout: id: 0400842c-f105-11ee-b647-cb9ed24678a4 2024-04-02T15:38:13.706 INFO:teuthology.orchestra.run.smithi087.stdout: health: HEALTH_OK 2024-04-02T15:38:13.706 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-02T15:38:13.706 INFO:teuthology.orchestra.run.smithi087.stdout: services: 2024-04-02T15:38:13.706 INFO:teuthology.orchestra.run.smithi087.stdout: mon: 3 daemons, quorum a,c,b (age 12m) 2024-04-02T15:38:13.706 INFO:teuthology.orchestra.run.smithi087.stdout: mgr: y(active, since 8m), standbys: x 2024-04-02T15:38:13.706 INFO:teuthology.orchestra.run.smithi087.stdout: osd: 8 osds: 8 up (since 8m), 8 in (since 9m) 2024-04-02T15:38:13.706 INFO:teuthology.orchestra.run.smithi087.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-04-02T15:38:13.706 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-02T15:38:13.706 INFO:teuthology.orchestra.run.smithi087.stdout: data: 2024-04-02T15:38:13.706 INFO:teuthology.orchestra.run.smithi087.stdout: pools: 6 pools, 161 pgs 2024-04-02T15:38:13.707 INFO:teuthology.orchestra.run.smithi087.stdout: objects: 209 objects, 585 KiB 2024-04-02T15:38:13.707 INFO:teuthology.orchestra.run.smithi087.stdout: usage: 63 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:38:13.707 INFO:teuthology.orchestra.run.smithi087.stdout: pgs: 161 active+clean 2024-04-02T15:38:13.707 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-02T15:38:13.707 INFO:teuthology.orchestra.run.smithi087.stdout: io: 2024-04-02T15:38:13.707 INFO:teuthology.orchestra.run.smithi087.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-04-02T15:38:13.707 INFO:teuthology.orchestra.run.smithi087.stdout: 2024-04-02T15:38:14.380 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph health detail' 2024-04-02T15:38:14.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:14 smithi087 ceph-mon[24900]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:38:14.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:14 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/3543418878' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-02T15:38:14.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:14 smithi087 ceph-mon[29707]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:38:14.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:14 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/3543418878' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-02T15:38:14.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:14 smithi106 ceph-mon[25843]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:38:14.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:14 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/3543418878' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-02T15:38:16.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:16 smithi087 ceph-mon[29707]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:38:16.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:16 smithi087 ceph-mon[24900]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:38:16.622 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:16 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:38:16] "GET /metrics HTTP/1.1" 200 207943 "" "Prometheus/2.33.4" 2024-04-02T15:38:16.802 INFO:teuthology.orchestra.run.smithi087.stdout:HEALTH_OK 2024-04-02T15:38:16.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:16 smithi106 ceph-mon[25843]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:38:17.450 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:17 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/2189611800' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T15:38:17.451 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:17 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/2189611800' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T15:38:17.475 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 2'"'"'' 2024-04-02T15:38:17.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:17 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/2189611800' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T15:38:18.799 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:18 smithi087 ceph-mon[29707]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:38:18.799 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:18 smithi087 ceph-mon[24900]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:38:18.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:18 smithi106 ceph-mon[25843]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:38:19.555 INFO:teuthology.orchestra.run.smithi087.stdout:true 2024-04-02T15:38:20.197 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph mgr fail' 2024-04-02T15:38:20.506 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:20 smithi087 ceph-mon[29707]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:38:20.507 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:20 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:38:20.507 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:20 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/1968103632' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T15:38:20.507 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:20 smithi087 ceph-mon[24900]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:38:20.507 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:20 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:38:20.507 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:20 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/1968103632' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T15:38:20.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:20 smithi106 ceph-mon[25843]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:38:20.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:20 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:38:20.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:20 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/1968103632' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T15:38:21.514 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:21 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 172.21.15.106 - - [02/Apr/2024:15:38:21] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-02T15:38:21.797 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:38:21 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:38:21.626Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:38:21.797 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:38:21 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:38:21.626Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:38:21.798 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:38:21 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:38:21.628Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:38:22.605 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:22 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:38:22.605 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:22 smithi087 ceph-mon[29707]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:38:22.605 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:22 smithi087 ceph-mon[29707]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:38:22.605 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:22 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/1343882530' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-02T15:38:22.605 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:22 smithi087 ceph-mon[29707]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-02T15:38:22.605 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:22 smithi087 ceph-mon[29707]: osdmap e96: 8 total, 8 up, 8 in 2024-04-02T15:38:22.605 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:22 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:38:22.295+0000 7ff93bf61700 -1 mgr handle_mgr_map I was active but no longer am 2024-04-02T15:38:22.606 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:22 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ignoring --setuser ceph since I am not root 2024-04-02T15:38:22.606 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:22 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ignoring --setgroup ceph since I am not root 2024-04-02T15:38:22.606 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:22 smithi087 ceph-mgr[25136]: -- 172.21.15.87:0/1810332560 <== mon.1 v2:172.21.15.87:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x563b5ec68340 con 0x563b5ec40c00 2024-04-02T15:38:22.606 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:22 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:38:22.606 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:22 smithi087 ceph-mon[24900]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:38:22.606 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:22 smithi087 ceph-mon[24900]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:38:22.606 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:22 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/1343882530' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-02T15:38:22.606 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:22 smithi087 ceph-mon[24900]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-02T15:38:22.606 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:22 smithi087 ceph-mon[24900]: osdmap e96: 8 total, 8 up, 8 in 2024-04-02T15:38:22.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:22 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:38:22.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:22 smithi106 ceph-mon[25843]: from='mgr.14448 172.21.15.87:0/4023517117' entity='mgr.y' 2024-04-02T15:38:22.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:22 smithi106 ceph-mon[25843]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:38:22.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:22 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/1343882530' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-02T15:38:22.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:22 smithi106 ceph-mon[25843]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-02T15:38:22.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:22 smithi106 ceph-mon[25843]: osdmap e96: 8 total, 8 up, 8 in 2024-04-02T15:38:22.871 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:22 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:38:22.809+0000 7fa764244000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-02T15:38:23.054 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'sleep 180' 2024-04-02T15:38:23.295 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:22 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:38:22.898+0000 7fa764244000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-02T15:38:23.295 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:38:22 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:38:22.931Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=7 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": dial tcp 172.21.15.87:8443: connect: connection refused" 2024-04-02T15:38:23.295 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:38:22 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:38:22.940Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=3 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": dial tcp 172.21.15.87:8443: connect: connection refused" 2024-04-02T15:38:23.295 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-mon[24900]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-02T15:38:23.295 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-mon[24900]: mgrmap e22: x(active, starting, since 0.0695978s) 2024-04-02T15:38:23.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:23 smithi106 ceph-mon[25843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-02T15:38:23.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:23 smithi106 ceph-mon[25843]: mgrmap e22: x(active, starting, since 0.0695978s) 2024-04-02T15:38:23.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:23 smithi106 ceph-mon[25843]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T15:38:23.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:23 smithi106 ceph-mon[25843]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:38:23.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:23 smithi106 ceph-mon[25843]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:38:23.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:23 smithi106 ceph-mon[25843]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-02T15:38:23.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:23 smithi106 ceph-mon[25843]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T15:38:23.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:23 smithi106 ceph-mon[25843]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T15:38:23.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:23 smithi106 ceph-mon[25843]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T15:38:23.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:23 smithi106 ceph-mon[25843]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T15:38:23.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:23 smithi106 ceph-mon[25843]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T15:38:23.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:23 smithi106 ceph-mon[25843]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T15:38:23.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:23 smithi106 ceph-mon[25843]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T15:38:23.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:23 smithi106 ceph-mon[25843]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:38:23.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:23 smithi106 ceph-mon[25843]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T15:38:23.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:23 smithi106 ceph-mon[25843]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T15:38:23.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:23 smithi106 ceph-mon[25843]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T15:38:23.619 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:23 smithi106 ceph-mon[25843]: Manager daemon x is now available 2024-04-02T15:38:23.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:38:23.542+0000 7fa764244000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-02T15:38:23.622 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:38:23.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": dial tcp 172.21.15.87:8443: connect: connection refused" 2024-04-02T15:38:23.622 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:38:23.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": dial tcp 172.21.15.87:8443: connect: connection refused" 2024-04-02T15:38:23.622 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:38:23.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:38:23.623 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-mon[24900]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T15:38:23.623 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-mon[24900]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:38:23.623 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-mon[24900]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:38:23.623 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-mon[24900]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-02T15:38:23.623 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-mon[24900]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T15:38:23.623 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-mon[24900]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T15:38:23.624 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-mon[24900]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T15:38:23.624 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-mon[24900]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T15:38:23.624 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-mon[24900]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T15:38:23.624 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-mon[24900]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T15:38:23.624 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-mon[24900]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T15:38:23.624 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-mon[24900]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:38:23.624 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-mon[24900]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T15:38:23.625 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-mon[24900]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T15:38:23.625 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-mon[24900]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T15:38:23.625 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-mon[24900]: Manager daemon x is now available 2024-04-02T15:38:23.625 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-mon[29707]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-02T15:38:23.626 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-mon[29707]: mgrmap e22: x(active, starting, since 0.0695978s) 2024-04-02T15:38:23.626 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-mon[29707]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-02T15:38:23.626 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-mon[29707]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-02T15:38:23.626 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-mon[29707]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-02T15:38:23.626 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-mon[29707]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-02T15:38:23.626 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-mon[29707]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T15:38:23.627 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-mon[29707]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T15:38:23.627 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-mon[29707]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T15:38:23.627 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-mon[29707]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T15:38:23.627 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-mon[29707]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T15:38:23.627 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-mon[29707]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T15:38:23.627 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-mon[29707]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-02T15:38:23.627 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-mon[29707]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-02T15:38:23.628 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-mon[29707]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T15:38:23.628 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-mon[29707]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T15:38:23.628 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-mon[29707]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T15:38:23.628 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-mon[29707]: Manager daemon x is now available 2024-04-02T15:38:23.950 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:38:23.632+0000 7fa764244000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-02T15:38:23.950 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:38:23.719+0000 7fa764244000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-02T15:38:23.950 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:38:23.869+0000 7fa764244000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-02T15:38:23.950 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:23 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:38:23.952+0000 7fa764244000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-02T15:38:24.300 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:24 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:38:24.062+0000 7fa764244000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-02T15:38:24.300 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:24 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:38:24.146+0000 7fa764244000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-02T15:38:24.568 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:24 smithi087 ceph-mon[29707]: mgrmap e23: x(active, since 1.07524s) 2024-04-02T15:38:24.568 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:24 smithi087 ceph-mon[29707]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:38:24.568 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:24 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:38:24.313+0000 7fa764244000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-02T15:38:24.569 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:24 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:38:24.400+0000 7fa764244000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-02T15:38:24.569 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:24 smithi087 ceph-mon[24900]: mgrmap e23: x(active, since 1.07524s) 2024-04-02T15:38:24.569 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:24 smithi087 ceph-mon[24900]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:38:24.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:24 smithi106 ceph-mon[25843]: mgrmap e23: x(active, since 1.07524s) 2024-04-02T15:38:24.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:24 smithi106 ceph-mon[25843]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:38:25.593 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:25 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:38:25.444+0000 7fa764244000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-02T15:38:25.593 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:25 smithi087 ceph-mon[29707]: Health check failed: 16 mgr modules have failed dependencies (MGR_MODULE_DEPENDENCY) 2024-04-02T15:38:25.593 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:25 smithi087 ceph-mon[29707]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:38:25.593 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:25 smithi087 ceph-mon[24900]: Health check failed: 16 mgr modules have failed dependencies (MGR_MODULE_DEPENDENCY) 2024-04-02T15:38:25.593 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:25 smithi087 ceph-mon[24900]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:38:25.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:25 smithi106 ceph-mon[25843]: Health check failed: 16 mgr modules have failed dependencies (MGR_MODULE_DEPENDENCY) 2024-04-02T15:38:25.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:25 smithi106 ceph-mon[25843]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:38:25.871 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:25 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:38:25.595+0000 7fa764244000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-02T15:38:25.871 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:25 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:38:25.792+0000 7fa764244000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-02T15:38:26.337 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:25 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:38:25.880+0000 7fa764244000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-02T15:38:26.337 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:26 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:38:26.015+0000 7fa764244000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-02T15:38:26.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:26 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:38:26.338+0000 7fa764244000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-02T15:38:26.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:26 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:38:26.557+0000 7fa764244000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-02T15:38:27.121 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:26 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:38:26.641+0000 7fa764244000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-02T15:38:27.121 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:26 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:38:26.809+0000 7fa764244000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-02T15:38:27.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:27 smithi106 ceph-mon[25843]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:38:27.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:27 smithi087 ceph-mon[24900]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:38:27.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:27 smithi087 ceph-mon[29707]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:38:28.017 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:27 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:38:27.727+0000 7fa764244000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-02T15:38:28.310 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:28 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:38:28.019+0000 7fa764244000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-02T15:38:28.310 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:28 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:38:28.115+0000 7fa764244000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-02T15:38:28.310 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:28 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:38:28.265+0000 7fa764244000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-02T15:38:28.310 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:28 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: [02/Apr/2024:15:38:28] ENGINE Bus STARTING 2024-04-02T15:38:28.582 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:28 smithi087 ceph-mon[29707]: Standby manager daemon y started 2024-04-02T15:38:28.582 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:28 smithi087 ceph-mon[29707]: from='mgr.? 172.21.15.87:0/3699912690' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-02T15:38:28.582 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:28 smithi087 ceph-mon[29707]: from='mgr.? 172.21.15.87:0/3699912690' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T15:38:28.582 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:28 smithi087 ceph-mon[29707]: from='mgr.? 172.21.15.87:0/3699912690' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-02T15:38:28.582 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:28 smithi087 ceph-mon[29707]: from='mgr.? 172.21.15.87:0/3699912690' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T15:38:28.583 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:28 smithi087 ceph-mon[24900]: Standby manager daemon y started 2024-04-02T15:38:28.583 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:28 smithi087 ceph-mon[24900]: from='mgr.? 172.21.15.87:0/3699912690' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-02T15:38:28.583 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:28 smithi087 ceph-mon[24900]: from='mgr.? 172.21.15.87:0/3699912690' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T15:38:28.583 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:28 smithi087 ceph-mon[24900]: from='mgr.? 172.21.15.87:0/3699912690' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-02T15:38:28.583 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:28 smithi087 ceph-mon[24900]: from='mgr.? 172.21.15.87:0/3699912690' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T15:38:28.583 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:28 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: CherryPy Checker: 2024-04-02T15:38:28.583 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:28 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: The Application mounted at '' has an empty config. 2024-04-02T15:38:28.583 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:28 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: 2024-04-02T15:38:28.583 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:28 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: [02/Apr/2024:15:38:28] ENGINE Serving on http://:::9283 2024-04-02T15:38:28.583 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:28 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: [02/Apr/2024:15:38:28] ENGINE Bus STARTED 2024-04-02T15:38:28.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:28 smithi106 ceph-mon[25843]: Standby manager daemon y started 2024-04-02T15:38:28.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:28 smithi106 ceph-mon[25843]: from='mgr.? 172.21.15.87:0/3699912690' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-02T15:38:28.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:28 smithi106 ceph-mon[25843]: from='mgr.? 172.21.15.87:0/3699912690' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T15:38:28.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:28 smithi106 ceph-mon[25843]: from='mgr.? 172.21.15.87:0/3699912690' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-02T15:38:28.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:28 smithi106 ceph-mon[25843]: from='mgr.? 172.21.15.87:0/3699912690' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T15:38:28.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:38:28 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:38:28.585Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=6 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:38:28.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:38:28 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:38:28.753Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=6 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:38:29.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:29 smithi106 ceph-mon[25843]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:38:29.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:29 smithi106 ceph-mon[25843]: mgrmap e24: x(active, since 6s), standbys: y 2024-04-02T15:38:29.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:29 smithi106 ceph-mon[25843]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T15:38:29.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:29 smithi087 ceph-mon[29707]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:38:29.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:29 smithi087 ceph-mon[29707]: mgrmap e24: x(active, since 6s), standbys: y 2024-04-02T15:38:29.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:29 smithi087 ceph-mon[29707]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T15:38:29.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:29 smithi087 ceph-mon[24900]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 67 MiB used, 715 GiB / 715 GiB avail 2024-04-02T15:38:29.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:29 smithi087 ceph-mon[24900]: mgrmap e24: x(active, since 6s), standbys: y 2024-04-02T15:38:29.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:29 smithi087 ceph-mon[24900]: from='mgr.24758 172.21.15.106:0/1896683228' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-02T15:38:30.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:30 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:38:30.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:30 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:38:30.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:30 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:38:31.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:31 smithi106 ceph-mon[25843]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 731 B/s rd, 0 op/s 2024-04-02T15:38:31.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:31 smithi087 ceph-mon[29707]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 731 B/s rd, 0 op/s 2024-04-02T15:38:31.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:31 smithi087 ceph-mon[24900]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 731 B/s rd, 0 op/s 2024-04-02T15:38:32.121 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:38:31 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:38:31.627Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:38:32.121 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:38:31 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:38:31.627Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:38:32.121 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:38:31 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:38:31.629Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:38:33.558 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:33 smithi087 ceph-mon[24900]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 569 B/s rd, 0 op/s 2024-04-02T15:38:33.559 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:33 smithi087 ceph-mon[29707]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 569 B/s rd, 0 op/s 2024-04-02T15:38:33.559 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:38:33 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:38:33.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:38:33.559 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:38:33 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:38:33.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:38:33.559 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:38:33 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:38:33.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:38:33.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:33 smithi106 ceph-mon[25843]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 569 B/s rd, 0 op/s 2024-04-02T15:38:35.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:35 smithi106 ceph-mon[25843]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 931 B/s rd, 0 op/s 2024-04-02T15:38:35.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:35 smithi087 ceph-mon[24900]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 931 B/s rd, 0 op/s 2024-04-02T15:38:35.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:35 smithi087 ceph-mon[29707]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 931 B/s rd, 0 op/s 2024-04-02T15:38:36.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:36 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:38:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:38:37.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:37 smithi106 ceph-mon[25843]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:38:37.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:37 smithi087 ceph-mon[29707]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:38:37.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:37 smithi087 ceph-mon[24900]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:38:39.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:39 smithi106 ceph-mon[25843]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:38:39.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:39 smithi087 ceph-mon[29707]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:38:39.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:39 smithi087 ceph-mon[24900]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:38:40.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:40 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:38:40.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:40 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:38:40.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:40 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:38:41.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:41 smithi106 ceph-mon[25843]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:38:41.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:41 smithi087 ceph-mon[29707]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:38:41.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:41 smithi087 ceph-mon[24900]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:38:42.121 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:38:41 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:38:41.628Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:38:42.121 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:38:41 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:38:41.628Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:38:42.121 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:38:41 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:38:41.630Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:38:43.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:43 smithi106 ceph-mon[25843]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:38:43.618 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:43 smithi087 ceph-mon[29707]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:38:43.619 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:43 smithi087 ceph-mon[24900]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:38:43.619 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:38:43 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:38:43.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:38:43.619 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:38:43 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:38:43.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:38:43.619 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:38:43 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:38:43.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:38:45.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:45 smithi106 ceph-mon[25843]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:38:45.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:45 smithi087 ceph-mon[29707]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:38:45.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:45 smithi087 ceph-mon[24900]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:38:46.541 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:46 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:38:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:38:47.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:47 smithi106 ceph-mon[25843]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:38:47.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:47 smithi087 ceph-mon[29707]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:38:47.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:47 smithi087 ceph-mon[24900]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:38:49.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:49 smithi106 ceph-mon[25843]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:38:49.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:49 smithi087 ceph-mon[29707]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:38:49.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:49 smithi087 ceph-mon[24900]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:38:50.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:50 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:38:50.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:50 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:38:50.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:50 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:38:51.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:51 smithi106 ceph-mon[25843]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:38:51.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:51 smithi087 ceph-mon[29707]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:38:51.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:51 smithi087 ceph-mon[24900]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:38:52.096 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:38:51 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:38:51.628Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:38:52.096 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:38:51 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:38:51.629Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:38:52.096 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:38:51 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:38:51.630Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:38:53.419 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:53 smithi087 ceph-mon[29707]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:38:53.419 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:53 smithi087 ceph-mon[24900]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:38:53.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:53 smithi106 ceph-mon[25843]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:38:53.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:38:53 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:38:53.517Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:38:53.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:38:53 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:38:53.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:38:53.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:38:53 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:38:53.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:38:55.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:55 smithi087 ceph-mon[24900]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:38:55.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:55 smithi087 ceph-mon[29707]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:38:55.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:55 smithi106 ceph-mon[25843]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:38:56.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:38:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:38:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:38:57.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:57 smithi087 ceph-mon[29707]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:38:57.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:57 smithi087 ceph-mon[24900]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:38:57.669 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:57 smithi106 ceph-mon[25843]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:38:59.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:38:59 smithi106 ceph-mon[25843]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:38:59.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:38:59 smithi087 ceph-mon[29707]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:38:59.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:38:59 smithi087 ceph-mon[24900]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:39:00.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:39:00 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:39:00.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:39:00 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:39:00.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:39:00 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:39:01.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:39:01 smithi106 ceph-mon[25843]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:01.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:39:01 smithi087 ceph-mon[29707]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:01.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:39:01 smithi087 ceph-mon[24900]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:01.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:39:01 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:39:01.629Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:39:01.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:39:01 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:39:01.630Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:39:01.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:39:01 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:39:01.631Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:39:03.700 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:39:03 smithi087 ceph-mon[29707]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:03.701 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:39:03 smithi087 ceph-mon[24900]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:03.701 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:39:03 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:39:03.517Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:39:03.701 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:39:03 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:39:03.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:39:03.701 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:39:03 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:39:03.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:39:03.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:39:03 smithi106 ceph-mon[25843]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:05.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:39:05 smithi106 ceph-mon[25843]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:39:05.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:39:05 smithi087 ceph-mon[24900]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:39:05.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:39:05 smithi087 ceph-mon[29707]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:39:06.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:39:06 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:39:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:39:07.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:39:07 smithi106 ceph-mon[25843]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:07.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:39:07 smithi087 ceph-mon[24900]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:07.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:39:07 smithi087 ceph-mon[29707]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:09.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:39:09 smithi106 ceph-mon[25843]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:39:09.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:39:09 smithi087 ceph-mon[29707]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:39:09.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:39:09 smithi087 ceph-mon[24900]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:39:10.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:39:10 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:39:10.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:39:10 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:39:10.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:39:10 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:39:11.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:39:11 smithi106 ceph-mon[25843]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:11.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:39:11 smithi087 ceph-mon[29707]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:11.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:39:11 smithi087 ceph-mon[24900]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:11.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:39:11 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:39:11.629Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:39:11.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:39:11 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:39:11.630Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:39:11.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:39:11 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:39:11.631Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:39:13.515 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:39:13 smithi087 ceph-mon[29707]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:13.516 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:39:13 smithi087 ceph-mon[24900]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:13.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:39:13 smithi106 ceph-mon[25843]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:13.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:39:13 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:39:13.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:39:13.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:39:13 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:39:13.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:39:13.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:39:13 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:39:13.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:39:15.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:39:15 smithi106 ceph-mon[25843]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:39:15.874 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:39:15 smithi087 ceph-mon[29707]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:39:15.874 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:39:15 smithi087 ceph-mon[24900]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:39:16.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:39:16 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:39:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:39:17.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:39:17 smithi106 ceph-mon[25843]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:17.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:39:17 smithi087 ceph-mon[24900]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:17.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:39:17 smithi087 ceph-mon[29707]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:19.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:39:19 smithi106 ceph-mon[25843]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:39:19.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:39:19 smithi087 ceph-mon[24900]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:39:19.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:39:19 smithi087 ceph-mon[29707]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:39:20.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:39:20 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:39:20.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:39:20 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:39:20.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:39:20 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:39:21.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:39:21 smithi106 ceph-mon[25843]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:21.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:39:21 smithi087 ceph-mon[24900]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:21.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:39:21 smithi087 ceph-mon[29707]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:21.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:39:21 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:39:21.630Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:39:21.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:39:21 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:39:21.630Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:39:21.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:39:21 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:39:21.632Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:39:23.781 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:39:23 smithi087 ceph-mon[29707]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:23.782 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:39:23 smithi087 ceph-mon[24900]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:23.782 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:39:23 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:39:23.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:39:23.782 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:39:23 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:39:23.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:39:23.782 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:39:23 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:39:23.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:39:23.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:39:23 smithi106 ceph-mon[25843]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:25.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:39:25 smithi106 ceph-mon[25843]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:39:25.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:39:25 smithi087 ceph-mon[29707]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:39:25.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:39:25 smithi087 ceph-mon[24900]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:39:26.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:39:26 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:39:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:39:27.595 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:39:27 smithi106 ceph-mon[25843]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:27.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:39:27 smithi087 ceph-mon[29707]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:27.872 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:39:27 smithi087 ceph-mon[24900]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:29.603 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:39:29 smithi106 ceph-mon[25843]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:39:29.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:39:29 smithi087 ceph-mon[29707]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:39:29.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:39:29 smithi087 ceph-mon[24900]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:39:30.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:39:30 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:39:30.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:39:30 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:39:30.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:39:30 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:39:31.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:39:31 smithi106 ceph-mon[25843]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:31.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:39:31 smithi087 ceph-mon[24900]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:31.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:39:31 smithi087 ceph-mon[29707]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:31.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:39:31 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:39:31.630Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:39:31.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:39:31 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:39:31.631Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:39:31.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:39:31 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:39:31.632Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:39:33.744 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:39:33 smithi087 ceph-mon[24900]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:33.745 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:39:33 smithi087 ceph-mon[29707]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:33.745 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:39:33 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:39:33.520Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:39:33.745 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:39:33 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:39:33.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:39:33.745 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:39:33 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:39:33.522Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:39:33.868 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:39:33 smithi106 ceph-mon[25843]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:35.561 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:39:35 smithi087 ceph-mon[24900]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:39:35.561 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:39:35 smithi087 ceph-mon[29707]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:39:35.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:39:35 smithi106 ceph-mon[25843]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:39:36.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:39:36 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:39:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:39:37.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:39:37 smithi106 ceph-mon[25843]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:37.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:39:37 smithi087 ceph-mon[29707]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:37.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:39:37 smithi087 ceph-mon[24900]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:39.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:39:39 smithi106 ceph-mon[25843]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:39:39.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:39:39 smithi087 ceph-mon[29707]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:39:39.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:39:39 smithi087 ceph-mon[24900]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:39:40.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:39:40 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:39:40.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:39:40 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:39:40.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:39:40 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:39:41.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:39:41 smithi106 ceph-mon[25843]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:41.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:39:41 smithi087 ceph-mon[29707]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:41.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:39:41 smithi087 ceph-mon[24900]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:41.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:39:41 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:39:41.630Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:39:41.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:39:41 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:39:41.631Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:39:41.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:39:41 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:39:41.632Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:39:43.863 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:39:43 smithi087 ceph-mon[24900]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:43.864 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:39:43 smithi087 ceph-mon[29707]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:43.864 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:39:43 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:39:43.520Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:39:43.864 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:39:43 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:39:43.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:39:43.864 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:39:43 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:39:43.523Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:39:43.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:39:43 smithi106 ceph-mon[25843]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:45.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:39:45 smithi106 ceph-mon[25843]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:39:45.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:39:45 smithi087 ceph-mon[29707]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:39:45.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:39:45 smithi087 ceph-mon[24900]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:39:46.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:39:46 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:39:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:39:47.772 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:39:47 smithi087 ceph-mon[24900]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:47.772 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:39:47 smithi087 ceph-mon[29707]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:47.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:39:47 smithi106 ceph-mon[25843]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:49.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:39:49 smithi106 ceph-mon[25843]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:39:49.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:39:49 smithi087 ceph-mon[24900]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:39:49.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:39:49 smithi087 ceph-mon[29707]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:39:50.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:39:50 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:39:50.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:39:50 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:39:50.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:39:50 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:39:51.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:39:51 smithi106 ceph-mon[25843]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:51.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:39:51 smithi087 ceph-mon[29707]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:51.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:39:51 smithi087 ceph-mon[24900]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:51.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:39:51 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:39:51.631Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:39:51.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:39:51 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:39:51.632Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:39:51.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:39:51 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:39:51.633Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:39:53.823 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:39:53 smithi087 ceph-mon[29707]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:53.824 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:39:53 smithi087 ceph-mon[24900]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:53.824 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:39:53 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:39:53.521Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:39:53.824 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:39:53 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:39:53.522Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:39:53.824 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:39:53 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:39:53.523Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:39:53.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:39:53 smithi106 ceph-mon[25843]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:55.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:39:55 smithi106 ceph-mon[25843]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:39:55.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:39:55 smithi087 ceph-mon[24900]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:39:55.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:39:55 smithi087 ceph-mon[29707]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:39:56.371 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:39:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:39:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:39:57.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:39:57 smithi106 ceph-mon[25843]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:57.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:39:57 smithi087 ceph-mon[29707]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:57.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:39:57 smithi087 ceph-mon[24900]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:39:59.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:39:59 smithi106 ceph-mon[25843]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:39:59.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:39:59 smithi087 ceph-mon[24900]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:39:59.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:39:59 smithi087 ceph-mon[29707]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:40:00.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:00 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:40:00.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:00 smithi106 ceph-mon[25843]: Health detail: HEALTH_WARN 16 mgr modules have failed dependencies 2024-04-02T15:40:00.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:00 smithi106 ceph-mon[25843]: [WRN] MGR_MODULE_DEPENDENCY: 16 mgr modules have failed dependencies 2024-04-02T15:40:00.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:00 smithi106 ceph-mon[25843]: Module 'balancer' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:00 smithi106 ceph-mon[25843]: Module 'cephadm' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:00 smithi106 ceph-mon[25843]: Module 'crash' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:00 smithi106 ceph-mon[25843]: Module 'dashboard' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:00 smithi106 ceph-mon[25843]: Module 'devicehealth' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:00 smithi106 ceph-mon[25843]: Module 'iostat' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:00 smithi106 ceph-mon[25843]: Module 'nfs' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:00 smithi106 ceph-mon[25843]: Module 'orchestrator' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:00 smithi106 ceph-mon[25843]: Module 'pg_autoscaler' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:00 smithi106 ceph-mon[25843]: Module 'progress' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:00 smithi106 ceph-mon[25843]: Module 'prometheus' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.619 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:00 smithi106 ceph-mon[25843]: Module 'rbd_support' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.619 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:00 smithi106 ceph-mon[25843]: Module 'restful' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.619 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:00 smithi106 ceph-mon[25843]: Module 'status' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.619 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:00 smithi106 ceph-mon[25843]: Module 'telemetry' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.619 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:00 smithi106 ceph-mon[25843]: Module 'volumes' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:00 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:40:00.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:00 smithi087 ceph-mon[29707]: Health detail: HEALTH_WARN 16 mgr modules have failed dependencies 2024-04-02T15:40:00.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:00 smithi087 ceph-mon[29707]: [WRN] MGR_MODULE_DEPENDENCY: 16 mgr modules have failed dependencies 2024-04-02T15:40:00.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:00 smithi087 ceph-mon[29707]: Module 'balancer' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:00 smithi087 ceph-mon[29707]: Module 'cephadm' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:00 smithi087 ceph-mon[29707]: Module 'crash' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:00 smithi087 ceph-mon[29707]: Module 'dashboard' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:00 smithi087 ceph-mon[29707]: Module 'devicehealth' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:00 smithi087 ceph-mon[29707]: Module 'iostat' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:00 smithi087 ceph-mon[29707]: Module 'nfs' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:00 smithi087 ceph-mon[29707]: Module 'orchestrator' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:00 smithi087 ceph-mon[29707]: Module 'pg_autoscaler' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:00 smithi087 ceph-mon[29707]: Module 'progress' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:00 smithi087 ceph-mon[29707]: Module 'prometheus' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:00 smithi087 ceph-mon[29707]: Module 'rbd_support' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:00 smithi087 ceph-mon[29707]: Module 'restful' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:00 smithi087 ceph-mon[29707]: Module 'status' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:00 smithi087 ceph-mon[29707]: Module 'telemetry' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.623 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:00 smithi087 ceph-mon[29707]: Module 'volumes' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.623 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:00 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:40:00.623 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:00 smithi087 ceph-mon[24900]: Health detail: HEALTH_WARN 16 mgr modules have failed dependencies 2024-04-02T15:40:00.623 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:00 smithi087 ceph-mon[24900]: [WRN] MGR_MODULE_DEPENDENCY: 16 mgr modules have failed dependencies 2024-04-02T15:40:00.623 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:00 smithi087 ceph-mon[24900]: Module 'balancer' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.623 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:00 smithi087 ceph-mon[24900]: Module 'cephadm' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.623 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:00 smithi087 ceph-mon[24900]: Module 'crash' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.623 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:00 smithi087 ceph-mon[24900]: Module 'dashboard' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.623 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:00 smithi087 ceph-mon[24900]: Module 'devicehealth' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.623 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:00 smithi087 ceph-mon[24900]: Module 'iostat' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.623 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:00 smithi087 ceph-mon[24900]: Module 'nfs' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.623 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:00 smithi087 ceph-mon[24900]: Module 'orchestrator' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.623 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:00 smithi087 ceph-mon[24900]: Module 'pg_autoscaler' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.623 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:00 smithi087 ceph-mon[24900]: Module 'progress' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.623 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:00 smithi087 ceph-mon[24900]: Module 'prometheus' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.624 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:00 smithi087 ceph-mon[24900]: Module 'rbd_support' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.624 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:00 smithi087 ceph-mon[24900]: Module 'restful' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.624 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:00 smithi087 ceph-mon[24900]: Module 'status' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.624 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:00 smithi087 ceph-mon[24900]: Module 'telemetry' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:00.624 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:00 smithi087 ceph-mon[24900]: Module 'volumes' has failed dependency: No module named 'mgr_module' 2024-04-02T15:40:01.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:01 smithi106 ceph-mon[25843]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:01.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:01 smithi087 ceph-mon[29707]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:01.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:01 smithi087 ceph-mon[24900]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:02.121 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:40:01 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:40:01.631Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:40:02.121 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:40:01 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:40:01.632Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:40:02.122 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:40:01 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:40:01.633Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:40:03.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:03 smithi106 ceph-mon[25843]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:03.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:03 smithi087 ceph-mon[29707]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:03.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:03 smithi087 ceph-mon[24900]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:03.622 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:40:03 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:40:03.522Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:40:03.622 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:40:03 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:40:03.523Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:40:03.622 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:40:03 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:40:03.524Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:40:05.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:05 smithi106 ceph-mon[25843]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:40:05.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:05 smithi087 ceph-mon[29707]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:40:05.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:05 smithi087 ceph-mon[24900]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:40:06.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:40:06 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:40:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:40:07.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:07 smithi106 ceph-mon[25843]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:07.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:07 smithi087 ceph-mon[29707]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:07.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:07 smithi087 ceph-mon[24900]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:09.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:09 smithi106 ceph-mon[25843]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:40:09.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:09 smithi087 ceph-mon[29707]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:40:09.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:09 smithi087 ceph-mon[24900]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:40:10.521 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:10 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:40:10.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:10 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:40:10.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:10 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:40:11.586 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:11 smithi106 ceph-mon[25843]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:11.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:11 smithi087 ceph-mon[29707]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:11.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:11 smithi087 ceph-mon[24900]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:12.121 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:40:11 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:40:11.632Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:40:12.122 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:40:11 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:40:11.632Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:40:12.122 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:40:11 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:40:11.633Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:40:13.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:13 smithi106 ceph-mon[25843]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:13.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:13 smithi087 ceph-mon[24900]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:13.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:13 smithi087 ceph-mon[29707]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:13.622 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:40:13 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:40:13.523Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:40:13.622 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:40:13 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:40:13.524Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:40:13.622 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:40:13 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:40:13.525Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:40:15.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:15 smithi106 ceph-mon[25843]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:40:15.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:15 smithi087 ceph-mon[29707]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:40:15.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:15 smithi087 ceph-mon[24900]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:40:16.622 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:40:16 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:40:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:40:17.371 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:17 smithi087 ceph-mon[29707]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:17.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:17 smithi087 ceph-mon[24900]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:17.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:17 smithi106 ceph-mon[25843]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:19.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:19 smithi106 ceph-mon[25843]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:40:19.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:19 smithi087 ceph-mon[24900]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:40:19.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:19 smithi087 ceph-mon[29707]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:40:20.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:20 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:40:20.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:20 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:40:20.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:20 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:40:21.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:21 smithi106 ceph-mon[25843]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:21.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:21 smithi087 ceph-mon[29707]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:21.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:21 smithi087 ceph-mon[24900]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:22.121 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:40:21 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:40:21.633Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:40:22.122 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:40:21 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:40:21.633Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:40:22.122 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:40:21 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:40:21.635Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:40:23.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:23 smithi106 ceph-mon[25843]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:23.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:23 smithi087 ceph-mon[29707]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:23.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:23 smithi087 ceph-mon[24900]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:23.622 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:40:23 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:40:23.524Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:40:23.622 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:40:23 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:40:23.524Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:40:23.622 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:40:23 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:40:23.525Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:40:25.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:25 smithi106 ceph-mon[25843]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:40:25.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:25 smithi087 ceph-mon[29707]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:40:25.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:25 smithi087 ceph-mon[24900]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:40:26.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:40:26 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:40:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:40:27.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:27 smithi106 ceph-mon[25843]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:27.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:27 smithi087 ceph-mon[29707]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:27.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:27 smithi087 ceph-mon[24900]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:29.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:29 smithi106 ceph-mon[25843]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:40:29.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:29 smithi087 ceph-mon[24900]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:40:29.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:29 smithi087 ceph-mon[29707]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:40:30.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:30 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:40:30.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:30 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:40:30.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:30 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:40:31.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:31 smithi106 ceph-mon[25843]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:31.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:31 smithi087 ceph-mon[29707]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:31.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:31 smithi087 ceph-mon[24900]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:32.126 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:40:31 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:40:31.634Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:40:32.126 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:40:31 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:40:31.634Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:40:32.127 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:40:31 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:40:31.636Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:40:33.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:33 smithi106 ceph-mon[25843]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:33.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:33 smithi087 ceph-mon[29707]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:33.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:33 smithi087 ceph-mon[24900]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:33.622 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:40:33 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:40:33.525Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:40:33.622 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:40:33 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:40:33.525Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:40:33.622 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:40:33 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:40:33.526Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:40:35.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:35 smithi106 ceph-mon[25843]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:40:35.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:35 smithi087 ceph-mon[29707]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:40:35.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:35 smithi087 ceph-mon[24900]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:40:36.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:40:36 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:40:36] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:40:37.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:37 smithi106 ceph-mon[25843]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:37.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:37 smithi087 ceph-mon[24900]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:37.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:37 smithi087 ceph-mon[29707]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:39.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:39 smithi106 ceph-mon[25843]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:40:39.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:39 smithi087 ceph-mon[24900]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:40:39.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:39 smithi087 ceph-mon[29707]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:40:40.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:40 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:40:40.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:40 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:40:40.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:40 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:40:41.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:41 smithi106 ceph-mon[25843]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:41.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:41 smithi087 ceph-mon[24900]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:41.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:41 smithi087 ceph-mon[29707]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:42.121 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:40:41 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:40:41.635Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:40:42.121 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:40:41 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:40:41.635Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:40:42.122 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:40:41 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:40:41.637Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:40:43.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:43 smithi106 ceph-mon[25843]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:43.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:43 smithi087 ceph-mon[24900]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:43.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:43 smithi087 ceph-mon[29707]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:43.622 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:40:43 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:40:43.525Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:40:43.622 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:40:43 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:40:43.526Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:40:43.622 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:40:43 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:40:43.527Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:40:45.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:45 smithi106 ceph-mon[25843]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:40:45.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:45 smithi087 ceph-mon[24900]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:40:45.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:45 smithi087 ceph-mon[29707]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:40:46.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:40:46 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:40:46] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:40:47.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:47 smithi106 ceph-mon[25843]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:47.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:47 smithi087 ceph-mon[29707]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:47.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:47 smithi087 ceph-mon[24900]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:49.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:49 smithi106 ceph-mon[25843]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:40:49.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:49 smithi087 ceph-mon[24900]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:40:49.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:49 smithi087 ceph-mon[29707]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:40:50.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:50 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:40:50.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:50 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:40:50.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:50 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:40:51.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:51 smithi087 ceph-mon[24900]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:51.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:51 smithi087 ceph-mon[29707]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:51.665 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:51 smithi106 ceph-mon[25843]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:52.122 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:40:51 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:40:51.636Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:40:52.122 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:40:51 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:40:51.636Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:40:52.122 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:40:51 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:40:51.638Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:40:53.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:53 smithi106 ceph-mon[25843]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:53.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:53 smithi087 ceph-mon[29707]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:53.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:53 smithi087 ceph-mon[24900]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:53.622 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:40:53 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:40:53.526Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:40:53.622 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:40:53 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:40:53.526Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:40:53.622 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:40:53 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:40:53.528Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:40:55.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:55 smithi106 ceph-mon[25843]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:40:55.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:55 smithi087 ceph-mon[24900]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:40:55.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:55 smithi087 ceph-mon[29707]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:40:56.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:40:56 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:40:56] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:40:57.587 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:57 smithi087 ceph-mon[29707]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:57.587 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:57 smithi087 ceph-mon[24900]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:57.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:57 smithi106 ceph-mon[25843]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:40:59.578 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:40:59 smithi087 ceph-mon[24900]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:40:59.578 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:40:59 smithi087 ceph-mon[29707]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:40:59.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:40:59 smithi106 ceph-mon[25843]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:41:00.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:41:00 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:41:00.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:41:00 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:41:00.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:41:00 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:41:01.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:41:01 smithi087 ceph-mon[24900]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:41:01.622 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:41:01 smithi087 ceph-mon[29707]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:41:01.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:41:01 smithi106 ceph-mon[25843]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:41:02.121 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:41:01 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:41:01.637Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:41:02.122 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:41:01 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:41:01.637Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:41:02.122 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:41:01 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:41:01.639Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:41:03.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:41:03 smithi106 ceph-mon[25843]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:41:03.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:41:03 smithi087 ceph-mon[29707]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:41:03.622 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:41:03 smithi087 ceph-mon[24900]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:41:03.622 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:41:03 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:41:03.526Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:41:03.622 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:41:03 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:41:03.527Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:41:03.623 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:41:03 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:41:03.528Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:41:05.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:41:05 smithi106 ceph-mon[25843]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:41:05.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:41:05 smithi087 ceph-mon[24900]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:41:05.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:41:05 smithi087 ceph-mon[29707]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:41:06.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:41:06 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:41:06] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:41:07.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:41:07 smithi106 ceph-mon[25843]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:41:07.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:41:07 smithi087 ceph-mon[24900]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:41:07.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:41:07 smithi087 ceph-mon[29707]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:41:09.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:41:09 smithi106 ceph-mon[25843]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:41:09.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:41:09 smithi087 ceph-mon[24900]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:41:09.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:41:09 smithi087 ceph-mon[29707]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:41:11.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:41:11 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:41:11.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:41:11 smithi106 ceph-mon[25843]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:41:11.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:41:11 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:41:11.872 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:41:11 smithi087 ceph-mon[24900]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:41:11.872 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:41:11 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:41:11.872 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:41:11 smithi087 ceph-mon[29707]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:41:11.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:41:11 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:41:11.637Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:41:11.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:41:11 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:41:11.638Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:41:11.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:41:11 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:41:11.639Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:41:13.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:41:13 smithi106 ceph-mon[25843]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:41:13.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:41:13 smithi087 ceph-mon[29707]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:41:13.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:41:13 smithi087 ceph-mon[24900]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:41:13.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:41:13 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:41:13.528Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:41:13.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:41:13 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:41:13.528Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:41:13.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:41:13 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:41:13.530Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:41:15.618 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:41:15 smithi106 ceph-mon[25843]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:41:15.621 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:41:15 smithi087 ceph-mon[24900]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:41:15.621 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:41:15 smithi087 ceph-mon[29707]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:41:16.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:41:16 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:41:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:41:17.683 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:41:17 smithi087 ceph-mon[29707]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:41:17.683 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:41:17 smithi087 ceph-mon[24900]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:41:17.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:41:17 smithi106 ceph-mon[25843]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:41:19.841 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:41:19 smithi087 ceph-mon[29707]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:41:19.841 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:41:19 smithi087 ceph-mon[24900]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:41:19.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:41:19 smithi106 ceph-mon[25843]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:41:21.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:41:21 smithi106 ceph-mon[25843]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:41:21.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:41:21 smithi106 ceph-mon[25843]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:41:21.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:41:21 smithi087 ceph-mon[29707]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:41:21.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:41:21 smithi087 ceph-mon[29707]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:41:21.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:41:21 smithi087 ceph-mon[24900]: from='client.24638 -' entity='client.iscsi.foo.smithi087.icnqbx' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-02T15:41:21.872 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:41:21 smithi087 ceph-mon[24900]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:41:21.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:41:21 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:41:21.638Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:41:21.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:41:21 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:41:21.638Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:41:21.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:41:21 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:41:21.640Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:41:23.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:41:23 smithi106 ceph-mon[25843]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:41:23.871 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:41:23 smithi087 ceph-mon[29707]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:41:23.871 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:41:23 smithi087 ceph-mon[24900]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:41:23.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:41:23 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:41:23.528Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:41:23.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:41:23 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:41:23.529Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:41:23.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:41:23 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:41:23.530Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:41:25.487 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:41:25 smithi087 ceph-mon[29707]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:41:25.488 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:41:25 smithi087 ceph-mon[24900]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:41:25.515 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-02T15:41:25.617 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:41:25 smithi106 ceph-mon[25843]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-02T15:41:26.371 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:41:26 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y[25132]: ::ffff:172.21.15.106 - - [02/Apr/2024:15:41:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-02T15:41:27.641 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:41:27 smithi087 ceph-mon[24900]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:41:27.641 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:41:27 smithi087 ceph-mon[29707]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:41:27.667 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:41:27 smithi106 ceph-mon[25843]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-02T15:41:27.728 INFO:teuthology.orchestra.run.smithi087.stderr:no valid command found; 10 closest matches: 2024-04-02T15:41:27.729 INFO:teuthology.orchestra.run.smithi087.stderr:pg stat 2024-04-02T15:41:27.729 INFO:teuthology.orchestra.run.smithi087.stderr:pg getmap 2024-04-02T15:41:27.729 INFO:teuthology.orchestra.run.smithi087.stderr:pg dump [...] 2024-04-02T15:41:27.729 INFO:teuthology.orchestra.run.smithi087.stderr:pg dump_json [...] 2024-04-02T15:41:27.729 INFO:teuthology.orchestra.run.smithi087.stderr:pg dump_pools_json 2024-04-02T15:41:27.729 INFO:teuthology.orchestra.run.smithi087.stderr:pg ls-by-pool [...] 2024-04-02T15:41:27.729 INFO:teuthology.orchestra.run.smithi087.stderr:pg ls-by-primary [] [...] 2024-04-02T15:41:27.729 INFO:teuthology.orchestra.run.smithi087.stderr:pg ls-by-osd [] [...] 2024-04-02T15:41:27.729 INFO:teuthology.orchestra.run.smithi087.stderr:pg ls [] [...] 2024-04-02T15:41:27.729 INFO:teuthology.orchestra.run.smithi087.stderr:pg dump_stuck [...] [] 2024-04-02T15:41:27.730 INFO:teuthology.orchestra.run.smithi087.stderr:Error EINVAL: invalid command 2024-04-02T15:41:28.494 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-04-02T15:41:28.495 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph-c_3f3f3ed992429b85be729743ad3b39d52dd90744/qa/tasks/cephadm.py", line 1459, in shell _shell( File "/home/teuthworker/src/git.ceph.com_ceph-c_3f3f3ed992429b85be729743ad3b39d52dd90744/qa/tasks/cephadm.py", line 110, in _shell return remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi087 with status 22: 'sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c \'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1\'' 2024-04-02T15:41:28.706 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=d8f30325fb784214b57dd646d1c9fe8f Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph-c_3f3f3ed992429b85be729743ad3b39d52dd90744/qa/tasks/cephadm.py", line 1459, in shell _shell( File "/home/teuthworker/src/git.ceph.com_ceph-c_3f3f3ed992429b85be729743ad3b39d52dd90744/qa/tasks/cephadm.py", line 110, in _shell return remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi087 with status 22: 'sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c \'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1\'' 2024-04-02T15:41:28.709 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-04-02T15:41:28.719 INFO:tasks.cephadm:Teardown begin 2024-04-02T15:41:28.720 DEBUG:teuthology.orchestra.run.smithi087:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-02T15:41:28.749 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:41:28 smithi087 ceph-mon[29707]: from='client.? 172.21.15.87:0/3401727304' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-02T15:41:28.749 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:41:28 smithi087 ceph-mon[24900]: from='client.? 172.21.15.87:0/3401727304' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-02T15:41:28.753 DEBUG:teuthology.orchestra.run.smithi106:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-02T15:41:28.781 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:41:28 smithi106 ceph-mon[25843]: from='client.? 172.21.15.87:0/3401727304' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-02T15:41:28.786 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-04-02T15:41:28.786 DEBUG:teuthology.orchestra.run.smithi087:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-02T15:41:28.810 DEBUG:teuthology.orchestra.run.smithi106:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-02T15:41:28.841 INFO:tasks.cephadm:Stopping all daemons... 2024-04-02T15:41:28.842 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-04-02T15:41:28.842 DEBUG:teuthology.orchestra.run.smithi087:> sudo systemctl stop ceph-0400842c-f105-11ee-b647-cb9ed24678a4@mon.a 2024-04-02T15:41:29.049 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:41:28 smithi087 systemd[1]: Stopping Ceph mon.a for 0400842c-f105-11ee-b647-cb9ed24678a4... 2024-04-02T15:41:29.049 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:41:28 smithi087 bash[64606]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mon.a" found: no such container 2024-04-02T15:41:29.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:41:29 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mon-a[24896]: 2024-04-02T15:41:29.049+0000 7f82464eb700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T15:41:29.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:41:29 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mon-a[24896]: 2024-04-02T15:41:29.049+0000 7f82464eb700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-04-02T15:41:29.372 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:41:29 smithi087 podman[64619]: 2024-04-02 15:41:29.121666793 +0000 UTC m=+0.195901581 container died e28fd32a7d9a16e9cd29d93249526e5ab8e7c89a1154253f4f465df687030731 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mon-a, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., version=8, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-02T15:41:29.857 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:41:29 smithi087 podman[64619]: 2024-04-02 15:41:29.591964704 +0000 UTC m=+0.666199491 container cleanup e28fd32a7d9a16e9cd29d93249526e5ab8e7c89a1154253f4f465df687030731 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mon-a, io.buildah.version=1.19.8, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, release=754, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, distribution-scope=public, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, architecture=x86_64) 2024-04-02T15:41:29.857 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:41:29 smithi087 bash[64619]: ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mon-a 2024-04-02T15:41:29.858 INFO:journalctl@ceph.mon.a.smithi087.stdout:Apr 02 15:41:29 smithi087 bash[64646]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mon.a" found: no such container 2024-04-02T15:41:29.863 DEBUG:teuthology.orchestra.run.smithi087:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0400842c-f105-11ee-b647-cb9ed24678a4@mon.a.service' 2024-04-02T15:41:29.909 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T15:41:29.909 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-04-02T15:41:29.910 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-04-02T15:41:29.910 DEBUG:teuthology.orchestra.run.smithi087:> sudo systemctl stop ceph-0400842c-f105-11ee-b647-cb9ed24678a4@mon.c 2024-04-02T15:41:30.121 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:41:30 smithi087 systemd[1]: Stopping Ceph mon.c for 0400842c-f105-11ee-b647-cb9ed24678a4... 2024-04-02T15:41:30.122 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:41:30 smithi087 bash[64701]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mon.c" found: no such container 2024-04-02T15:41:30.410 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:41:30 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mon-c[29689]: 2024-04-02T15:41:30.146+0000 7fd92e766700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T15:41:30.410 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:41:30 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mon-c[29689]: 2024-04-02T15:41:30.146+0000 7fd92e766700 -1 mon.c@1(peon) e3 *** Got Signal Terminated *** 2024-04-02T15:41:30.704 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:41:30 smithi087 podman[64715]: 2024-04-02 15:41:30.412837464 +0000 UTC m=+0.383452250 container died 40f23934618c40010db6cb10ca7763742ac86a7b63337e73ae3f5c1fb728ea81 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mon-c, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, io.openshift.expose-services=, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, io.buildah.version=1.19.8, ceph=True, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8) 2024-04-02T15:41:30.956 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:41:30 smithi087 podman[64715]: 2024-04-02 15:41:30.922237445 +0000 UTC m=+0.892852215 container cleanup 40f23934618c40010db6cb10ca7763742ac86a7b63337e73ae3f5c1fb728ea81 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mon-c, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, version=8, vendor=Red Hat, Inc., io.buildah.version=1.19.8, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, name=centos-stream, ceph=True, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, io.openshift.expose-services=, distribution-scope=public, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, io.openshift.tags=base centos centos-stream) 2024-04-02T15:41:30.956 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:41:30 smithi087 bash[64715]: ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mon-c 2024-04-02T15:41:30.956 INFO:journalctl@ceph.mon.c.smithi087.stdout:Apr 02 15:41:30 smithi087 bash[64742]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mon.c" found: no such container 2024-04-02T15:41:31.152 DEBUG:teuthology.orchestra.run.smithi087:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0400842c-f105-11ee-b647-cb9ed24678a4@mon.c.service' 2024-04-02T15:41:31.194 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T15:41:31.194 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-04-02T15:41:31.194 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-04-02T15:41:31.195 DEBUG:teuthology.orchestra.run.smithi106:> sudo systemctl stop ceph-0400842c-f105-11ee-b647-cb9ed24678a4@mon.b 2024-04-02T15:41:31.478 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:41:31 smithi106 systemd[1]: Stopping Ceph mon.b for 0400842c-f105-11ee-b647-cb9ed24678a4... 2024-04-02T15:41:31.478 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:41:31 smithi106 bash[54362]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mon.b" found: no such container 2024-04-02T15:41:31.478 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:41:31 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mon-b[25839]: 2024-04-02T15:41:31.391+0000 7f5dc7c7d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T15:41:31.478 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:41:31 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mon-b[25839]: 2024-04-02T15:41:31.391+0000 7f5dc7c7d700 -1 mon.b@2(peon) e3 *** Got Signal Terminated *** 2024-04-02T15:41:31.867 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:41:31 smithi106 podman[54375]: 2024-04-02 15:41:31.595890685 +0000 UTC m=+0.310761839 container died d9de5ac6a73e6f45899b23f50d1cac0fe7f5d0af2f64640f7348af83fe702801 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mon-b, vendor=Red Hat, Inc., GIT_CLEAN=True, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, release=754, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, version=8, io.openshift.expose-services=) 2024-04-02T15:41:32.121 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:41:31 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:41:31.638Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:41:32.122 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:41:31 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:41:31.638Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:41:32.122 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:41:31 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:41:31.640Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.87 because it doesn't contain any IP SANs" 2024-04-02T15:41:32.191 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:41:32 smithi106 podman[54375]: 2024-04-02 15:41:32.025292551 +0000 UTC m=+0.740163693 container cleanup d9de5ac6a73e6f45899b23f50d1cac0fe7f5d0af2f64640f7348af83fe702801 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mon-b, GIT_BRANCH=HEAD, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, io.openshift.expose-services=, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, vcs-type=git, version=8, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, com.redhat.component=centos-stream-container) 2024-04-02T15:41:32.191 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:41:32 smithi106 bash[54375]: ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mon-b 2024-04-02T15:41:32.191 INFO:journalctl@ceph.mon.b.smithi106.stdout:Apr 02 15:41:32 smithi106 bash[54488]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mon.b" found: no such container 2024-04-02T15:41:32.196 DEBUG:teuthology.orchestra.run.smithi106:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0400842c-f105-11ee-b647-cb9ed24678a4@mon.b.service' 2024-04-02T15:41:32.244 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T15:41:32.244 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-04-02T15:41:32.244 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-04-02T15:41:32.244 DEBUG:teuthology.orchestra.run.smithi087:> sudo systemctl stop ceph-0400842c-f105-11ee-b647-cb9ed24678a4@mgr.y 2024-04-02T15:41:32.621 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:41:32 smithi087 systemd[1]: Stopping Ceph mgr.y for 0400842c-f105-11ee-b647-cb9ed24678a4... 2024-04-02T15:41:32.622 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:41:32 smithi087 bash[64795]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr.y" found: no such container 2024-04-02T15:41:32.622 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:41:32 smithi087 podman[64808]: 2024-04-02 15:41:32.499174353 +0000 UTC m=+0.169664232 container died 5d422ff3cc7d951a9ea798f2aa50cf7abb9d711f0fdd5d165830165c8dc0ed88 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, vcs-type=git, GIT_BRANCH=HEAD, architecture=x86_64, io.openshift.tags=base centos centos-stream, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, ceph=True, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, RELEASE=HEAD, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True) 2024-04-02T15:41:33.050 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:41:32 smithi087 podman[64808]: 2024-04-02 15:41:32.843222731 +0000 UTC m=+0.513712596 container cleanup 5d422ff3cc7d951a9ea798f2aa50cf7abb9d711f0fdd5d165830165c8dc0ed88 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, io.buildah.version=1.19.8, distribution-scope=public, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , RELEASE=HEAD, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc.) 2024-04-02T15:41:33.050 INFO:journalctl@ceph.mgr.y.smithi087.stdout:Apr 02 15:41:32 smithi087 bash[64808]: ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-y 2024-04-02T15:41:33.119 DEBUG:teuthology.orchestra.run.smithi087:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0400842c-f105-11ee-b647-cb9ed24678a4@mgr.y.service' 2024-04-02T15:41:33.156 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T15:41:33.156 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-04-02T15:41:33.156 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-04-02T15:41:33.156 DEBUG:teuthology.orchestra.run.smithi106:> sudo systemctl stop ceph-0400842c-f105-11ee-b647-cb9ed24678a4@mgr.x 2024-04-02T15:41:33.339 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:41:33 smithi106 systemd[1]: Stopping Ceph mgr.x for 0400842c-f105-11ee-b647-cb9ed24678a4... 2024-04-02T15:41:33.371 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:41:33 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:41:33.113Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=7 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": dial tcp 172.21.15.87:8443: connect: connection refused" 2024-04-02T15:41:33.371 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:41:33 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:41:33.161Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=3 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": dial tcp 172.21.15.87:8443: connect: connection refused" 2024-04-02T15:41:33.617 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:41:33 smithi106 bash[54627]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr.x" found: no such container 2024-04-02T15:41:33.617 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:41:33 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x[50669]: 2024-04-02T15:41:33.477+0000 7f5746103700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.x -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T15:41:33.618 INFO:journalctl@ceph.mgr.x.smithi106.stdout:Apr 02 15:41:33 smithi106 podman[54641]: 2024-04-02 15:41:33.488920551 +0000 UTC m=+0.129855050 container died 94f3e590724e5a9967dfe923af1dfe5fbf01c74658694eac3c849aa0f22d0920 (image=quay.ceph.io/ceph-ci/ceph@sha256:6c740d4757bcfc646ba2af58c0d4aaf274efb0f36ed5b987df90c88e7e38aeac, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-mgr-x, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-testing-2024-04-02-0703-3f3f3ed, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2) 2024-04-02T15:41:33.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:41:33 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:41:33.529Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": dial tcp 172.21.15.87:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:41:33.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:41:33 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:41:33.530Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": dial tcp 172.21.15.87:8443: connect: connection refused" 2024-04-02T15:41:33.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:41:33 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:41:33.530Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:41:34.074 DEBUG:teuthology.orchestra.run.smithi106:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0400842c-f105-11ee-b647-cb9ed24678a4@mgr.x.service' 2024-04-02T15:41:34.117 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T15:41:34.117 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-04-02T15:41:34.117 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-04-02T15:41:34.117 DEBUG:teuthology.orchestra.run.smithi087:> sudo systemctl stop ceph-0400842c-f105-11ee-b647-cb9ed24678a4@osd.0 2024-04-02T15:41:34.622 INFO:journalctl@ceph.osd.0.smithi087.stdout:Apr 02 15:41:34 smithi087 systemd[1]: Stopping Ceph osd.0 for 0400842c-f105-11ee-b647-cb9ed24678a4... 2024-04-02T15:41:34.622 INFO:journalctl@ceph.osd.0.smithi087.stdout:Apr 02 15:41:34 smithi087 bash[64889]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd.0" found: no such container 2024-04-02T15:41:34.622 INFO:journalctl@ceph.osd.0.smithi087.stdout:Apr 02 15:41:34 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-0[33084]: 2024-04-02T15:41:34.314+0000 7f13ac7ff700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T15:41:34.622 INFO:journalctl@ceph.osd.0.smithi087.stdout:Apr 02 15:41:34 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-0[33084]: 2024-04-02T15:41:34.314+0000 7f13ac7ff700 -1 osd.0 96 *** Got signal Terminated *** 2024-04-02T15:41:34.622 INFO:journalctl@ceph.osd.0.smithi087.stdout:Apr 02 15:41:34 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-0[33084]: 2024-04-02T15:41:34.314+0000 7f13ac7ff700 -1 osd.0 96 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T15:41:40.114 INFO:journalctl@ceph.osd.0.smithi087.stdout:Apr 02 15:41:39 smithi087 podman[64902]: 2024-04-02 15:41:39.759345575 +0000 UTC m=+5.559760503 container died 644fbe6ff2bccd1951840ac8a4a16eb9aae3eb1cdbf1b07fc15acc952d37f88a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-0, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., architecture=x86_64, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, ceph=True, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8) 2024-04-02T15:41:40.365 INFO:journalctl@ceph.osd.0.smithi087.stdout:Apr 02 15:41:40 smithi087 podman[64902]: 2024-04-02 15:41:40.116133036 +0000 UTC m=+5.916547963 container cleanup 644fbe6ff2bccd1951840ac8a4a16eb9aae3eb1cdbf1b07fc15acc952d37f88a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , RELEASE=HEAD, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, ceph=True, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, architecture=x86_64, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, CEPH_POINT_RELEASE=-17.2.0, release=754, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream) 2024-04-02T15:41:40.365 INFO:journalctl@ceph.osd.0.smithi087.stdout:Apr 02 15:41:40 smithi087 bash[64902]: ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-0 2024-04-02T15:41:40.365 INFO:journalctl@ceph.osd.0.smithi087.stdout:Apr 02 15:41:40 smithi087 bash[65014]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd.0" found: no such container 2024-04-02T15:41:40.621 INFO:journalctl@ceph.osd.0.smithi087.stdout:Apr 02 15:41:40 smithi087 podman[64983]: 2024-04-02 15:41:40.426143315 +0000 UTC m=+0.660840051 container remove 644fbe6ff2bccd1951840ac8a4a16eb9aae3eb1cdbf1b07fc15acc952d37f88a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , RELEASE=HEAD, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., distribution-scope=public, com.redhat.component=centos-stream-container, io.openshift.expose-services=, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, release=754, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, version=8) 2024-04-02T15:41:41.363 INFO:journalctl@ceph.osd.0.smithi087.stdout:Apr 02 15:41:41 smithi087 podman[65119]: 2024-04-02 15:41:41.133706198 +0000 UTC m=+0.018054608 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-02T15:41:41.364 INFO:journalctl@ceph.osd.0.smithi087.stdout:Apr 02 15:41:41 smithi087 podman[65119]: 2024-04-02 15:41:41.239399841 +0000 UTC m=+0.123748244 container create adf413d7d51805f5f17fd5172f09f373d826c7ba44f9faaa370328c9222e3caa (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-0-deactivate, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, version=8, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container) 2024-04-02T15:41:41.621 INFO:journalctl@ceph.osd.0.smithi087.stdout:Apr 02 15:41:41 smithi087 podman[65119]: 2024-04-02 15:41:41.475210041 +0000 UTC m=+0.359558460 container init adf413d7d51805f5f17fd5172f09f373d826c7ba44f9faaa370328c9222e3caa (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-0-deactivate, maintainer=Guillaume Abrioux , name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, architecture=x86_64, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, version=8, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, ceph=True, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, GIT_BRANCH=HEAD) 2024-04-02T15:41:41.622 INFO:journalctl@ceph.osd.0.smithi087.stdout:Apr 02 15:41:41 smithi087 podman[65119]: 2024-04-02 15:41:41.478399209 +0000 UTC m=+0.362747612 container start adf413d7d51805f5f17fd5172f09f373d826c7ba44f9faaa370328c9222e3caa (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-0-deactivate, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, name=centos-stream, vcs-type=git, ceph=True, com.redhat.component=centos-stream-container, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=) 2024-04-02T15:41:41.622 INFO:journalctl@ceph.osd.0.smithi087.stdout:Apr 02 15:41:41 smithi087 podman[65119]: 2024-04-02 15:41:41.575393345 +0000 UTC m=+0.459741761 container attach adf413d7d51805f5f17fd5172f09f373d826c7ba44f9faaa370328c9222e3caa (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-0-deactivate, release=754, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, GIT_CLEAN=True, name=centos-stream, com.redhat.component=centos-stream-container, ceph=True, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, io.openshift.expose-services=, distribution-scope=public) 2024-04-02T15:41:42.068 INFO:journalctl@ceph.osd.0.smithi087.stdout:Apr 02 15:41:41 smithi087 podman[65119]: 2024-04-02 15:41:41.763392475 +0000 UTC m=+0.647740886 container died adf413d7d51805f5f17fd5172f09f373d826c7ba44f9faaa370328c9222e3caa (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-0-deactivate, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, version=8, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, io.openshift.expose-services=, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, architecture=x86_64) 2024-04-02T15:41:42.068 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:41:41 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:41:41.638Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": dial tcp 172.21.15.87:8443: connect: connection refused" 2024-04-02T15:41:42.068 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:41:41 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:41:41.639Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": dial tcp 172.21.15.87:8443: connect: connection refused" 2024-04-02T15:41:42.069 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:41:41 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:41:41.639Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:41:42.398 DEBUG:teuthology.orchestra.run.smithi087:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0400842c-f105-11ee-b647-cb9ed24678a4@osd.0.service' 2024-04-02T15:41:42.447 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T15:41:42.447 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-04-02T15:41:42.448 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-04-02T15:41:42.448 DEBUG:teuthology.orchestra.run.smithi087:> sudo systemctl stop ceph-0400842c-f105-11ee-b647-cb9ed24678a4@osd.1 2024-04-02T15:41:42.871 INFO:journalctl@ceph.osd.1.smithi087.stdout:Apr 02 15:41:42 smithi087 systemd[1]: Stopping Ceph osd.1 for 0400842c-f105-11ee-b647-cb9ed24678a4... 2024-04-02T15:41:42.872 INFO:journalctl@ceph.osd.1.smithi087.stdout:Apr 02 15:41:42 smithi087 bash[65199]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd.1" found: no such container 2024-04-02T15:41:42.872 INFO:journalctl@ceph.osd.1.smithi087.stdout:Apr 02 15:41:42 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-1[35962]: 2024-04-02T15:41:42.623+0000 7fa43aa43700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T15:41:42.872 INFO:journalctl@ceph.osd.1.smithi087.stdout:Apr 02 15:41:42 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-1[35962]: 2024-04-02T15:41:42.623+0000 7fa43aa43700 -1 osd.1 96 *** Got signal Terminated *** 2024-04-02T15:41:42.872 INFO:journalctl@ceph.osd.1.smithi087.stdout:Apr 02 15:41:42 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-1[35962]: 2024-04-02T15:41:42.623+0000 7fa43aa43700 -1 osd.1 96 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T15:41:43.875 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:41:43 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:41:43.530Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": dial tcp 172.21.15.87:8443: connect: connection refused" 2024-04-02T15:41:43.875 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:41:43 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:41:43.531Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": dial tcp 172.21.15.87:8443: connect: connection refused" 2024-04-02T15:41:43.875 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:41:43 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:41:43.531Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:41:48.226 INFO:journalctl@ceph.osd.1.smithi087.stdout:Apr 02 15:41:48 smithi087 podman[65212]: 2024-04-02 15:41:48.083812985 +0000 UTC m=+5.551385053 container died 8996e84013c298de205e2a03ca478ecef93ebba87f346d7be2c6bda667b2fce6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-1, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, name=centos-stream, release=754, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, RELEASE=HEAD, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream) 2024-04-02T15:41:48.871 INFO:journalctl@ceph.osd.1.smithi087.stdout:Apr 02 15:41:48 smithi087 podman[65212]: 2024-04-02 15:41:48.582530512 +0000 UTC m=+6.050102614 container cleanup 8996e84013c298de205e2a03ca478ecef93ebba87f346d7be2c6bda667b2fce6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-1, release=754, GIT_BRANCH=HEAD, RELEASE=HEAD, GIT_CLEAN=True, name=centos-stream, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, version=8, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, distribution-scope=public) 2024-04-02T15:41:48.872 INFO:journalctl@ceph.osd.1.smithi087.stdout:Apr 02 15:41:48 smithi087 bash[65212]: ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-1 2024-04-02T15:41:48.872 INFO:journalctl@ceph.osd.1.smithi087.stdout:Apr 02 15:41:48 smithi087 bash[65382]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd.1" found: no such container 2024-04-02T15:41:48.872 INFO:journalctl@ceph.osd.1.smithi087.stdout:Apr 02 15:41:48 smithi087 podman[65359]: 2024-04-02 15:41:48.817190247 +0000 UTC m=+0.731702792 container remove 8996e84013c298de205e2a03ca478ecef93ebba87f346d7be2c6bda667b2fce6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-1, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, io.openshift.expose-services=, release=754, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, GIT_CLEAN=True, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, name=centos-stream, build-date=2022-05-03T08:36:31.336870) 2024-04-02T15:41:49.318 INFO:journalctl@ceph.osd.1.smithi087.stdout:Apr 02 15:41:49 smithi087 podman[65463]: 2024-04-02 15:41:48.915261451 +0000 UTC m=+0.019215863 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-02T15:41:49.318 INFO:journalctl@ceph.osd.1.smithi087.stdout:Apr 02 15:41:49 smithi087 podman[65463]: 2024-04-02 15:41:49.018352067 +0000 UTC m=+0.122306513 container create a1f6eb6052dcf91da51751b1a450956cc0633a01a74d98d1201b3c2e74c5e3b8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-1-deactivate, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, build-date=2022-05-03T08:36:31.336870, vcs-type=git, GIT_BRANCH=HEAD, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, io.openshift.expose-services=, architecture=x86_64, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, ceph=True, version=8, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-04-02T15:41:49.319 INFO:journalctl@ceph.osd.1.smithi087.stdout:Apr 02 15:41:49 smithi087 podman[65463]: 2024-04-02 15:41:49.221077662 +0000 UTC m=+0.325032063 container init a1f6eb6052dcf91da51751b1a450956cc0633a01a74d98d1201b3c2e74c5e3b8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-1-deactivate, version=8, io.openshift.expose-services=, vcs-type=git, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, GIT_CLEAN=True, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-02T15:41:49.319 INFO:journalctl@ceph.osd.1.smithi087.stdout:Apr 02 15:41:49 smithi087 podman[65463]: 2024-04-02 15:41:49.224832904 +0000 UTC m=+0.328787309 container start a1f6eb6052dcf91da51751b1a450956cc0633a01a74d98d1201b3c2e74c5e3b8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-1-deactivate, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, version=8, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, ceph=True, maintainer=Guillaume Abrioux , RELEASE=HEAD, architecture=x86_64, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, distribution-scope=public) 2024-04-02T15:41:49.319 INFO:journalctl@ceph.osd.1.smithi087.stdout:Apr 02 15:41:49 smithi087 podman[65463]: 2024-04-02 15:41:49.320851255 +0000 UTC m=+0.424805656 container attach a1f6eb6052dcf91da51751b1a450956cc0633a01a74d98d1201b3c2e74c5e3b8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-1-deactivate, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, version=8, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, name=centos-stream, io.openshift.expose-services=, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, GIT_CLEAN=True, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64) 2024-04-02T15:41:49.621 INFO:journalctl@ceph.osd.1.smithi087.stdout:Apr 02 15:41:49 smithi087 podman[65463]: 2024-04-02 15:41:49.552009993 +0000 UTC m=+0.655964393 container died a1f6eb6052dcf91da51751b1a450956cc0633a01a74d98d1201b3c2e74c5e3b8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-1-deactivate, io.buildah.version=1.19.8, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, GIT_CLEAN=True, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, distribution-scope=public, RELEASE=HEAD) 2024-04-02T15:41:50.299 DEBUG:teuthology.orchestra.run.smithi087:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0400842c-f105-11ee-b647-cb9ed24678a4@osd.1.service' 2024-04-02T15:41:50.340 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T15:41:50.341 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-04-02T15:41:50.341 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-04-02T15:41:50.341 DEBUG:teuthology.orchestra.run.smithi087:> sudo systemctl stop ceph-0400842c-f105-11ee-b647-cb9ed24678a4@osd.2 2024-04-02T15:41:50.567 INFO:journalctl@ceph.osd.2.smithi087.stdout:Apr 02 15:41:50 smithi087 systemd[1]: Stopping Ceph osd.2 for 0400842c-f105-11ee-b647-cb9ed24678a4... 2024-04-02T15:41:50.567 INFO:journalctl@ceph.osd.2.smithi087.stdout:Apr 02 15:41:50 smithi087 bash[65587]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd.2" found: no such container 2024-04-02T15:41:50.871 INFO:journalctl@ceph.osd.2.smithi087.stdout:Apr 02 15:41:50 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-2[38901]: 2024-04-02T15:41:50.567+0000 7fa5427e7700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T15:41:50.871 INFO:journalctl@ceph.osd.2.smithi087.stdout:Apr 02 15:41:50 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-2[38901]: 2024-04-02T15:41:50.567+0000 7fa5427e7700 -1 osd.2 96 *** Got signal Terminated *** 2024-04-02T15:41:50.871 INFO:journalctl@ceph.osd.2.smithi087.stdout:Apr 02 15:41:50 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-2[38901]: 2024-04-02T15:41:50.567+0000 7fa5427e7700 -1 osd.2 96 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T15:41:51.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:41:51 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:41:51.639Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": dial tcp 172.21.15.87:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:41:51.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:41:51 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:41:51.639Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": dial tcp 172.21.15.87:8443: connect: connection refused" 2024-04-02T15:41:51.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:41:51 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:41:51.639Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:41:53.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:41:53 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:41:53.531Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": dial tcp 172.21.15.87:8443: connect: connection refused" 2024-04-02T15:41:53.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:41:53 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:41:53.532Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": dial tcp 172.21.15.87:8443: connect: connection refused" 2024-04-02T15:41:53.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:41:53 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:41:53.532Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:41:56.214 INFO:journalctl@ceph.osd.2.smithi087.stdout:Apr 02 15:41:55 smithi087 podman[65606]: 2024-04-02 15:41:55.96264667 +0000 UTC m=+5.501444731 container died 944a519c2f4b114ebc9fb0b0ac365b23e923b3de628a6460b390c510ed80a01f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-2, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, architecture=x86_64, vendor=Red Hat, Inc., version=8, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, com.redhat.component=centos-stream-container, io.openshift.expose-services=, maintainer=Guillaume Abrioux , vcs-type=git, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-04-02T15:41:56.489 INFO:journalctl@ceph.osd.2.smithi087.stdout:Apr 02 15:41:56 smithi087 podman[65606]: 2024-04-02 15:41:56.290417228 +0000 UTC m=+5.829215288 container cleanup 944a519c2f4b114ebc9fb0b0ac365b23e923b3de628a6460b390c510ed80a01f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-2, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, architecture=x86_64, name=centos-stream, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, GIT_BRANCH=HEAD, io.openshift.expose-services=, RELEASE=HEAD, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, vcs-type=git, build-date=2022-05-03T08:36:31.336870) 2024-04-02T15:41:56.489 INFO:journalctl@ceph.osd.2.smithi087.stdout:Apr 02 15:41:56 smithi087 bash[65606]: ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-2 2024-04-02T15:41:56.489 INFO:journalctl@ceph.osd.2.smithi087.stdout:Apr 02 15:41:56 smithi087 bash[66067]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd.2" found: no such container 2024-04-02T15:41:56.826 INFO:journalctl@ceph.osd.2.smithi087.stdout:Apr 02 15:41:56 smithi087 podman[66055]: 2024-04-02 15:41:56.491464013 +0000 UTC m=+0.526464892 container remove 944a519c2f4b114ebc9fb0b0ac365b23e923b3de628a6460b390c510ed80a01f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-2, release=754, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, io.openshift.expose-services=, vcs-type=git, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, distribution-scope=public, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True) 2024-04-02T15:41:56.827 INFO:journalctl@ceph.osd.2.smithi087.stdout:Apr 02 15:41:56 smithi087 podman[66133]: 2024-04-02 15:41:56.613498907 +0000 UTC m=+0.019046232 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-02T15:41:56.827 INFO:journalctl@ceph.osd.2.smithi087.stdout:Apr 02 15:41:56 smithi087 podman[66133]: 2024-04-02 15:41:56.717354149 +0000 UTC m=+0.122901454 container create f26624e9babfea270556d82570f1734a4373ac49d98dedbc6df29b62dcf6e411 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-2-deactivate, vendor=Red Hat, Inc., release=754, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, architecture=x86_64, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, name=centos-stream, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, version=8, com.redhat.component=centos-stream-container) 2024-04-02T15:41:57.121 INFO:journalctl@ceph.osd.2.smithi087.stdout:Apr 02 15:41:56 smithi087 podman[66133]: 2024-04-02 15:41:56.961892187 +0000 UTC m=+0.367439507 container init f26624e9babfea270556d82570f1734a4373ac49d98dedbc6df29b62dcf6e411 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-2-deactivate, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, GIT_CLEAN=True, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , version=8, name=centos-stream, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, architecture=x86_64) 2024-04-02T15:41:57.121 INFO:journalctl@ceph.osd.2.smithi087.stdout:Apr 02 15:41:56 smithi087 podman[66133]: 2024-04-02 15:41:56.964994273 +0000 UTC m=+0.370541582 container start f26624e9babfea270556d82570f1734a4373ac49d98dedbc6df29b62dcf6e411 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-2-deactivate, version=8, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, release=754, vcs-type=git, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8) 2024-04-02T15:41:57.122 INFO:journalctl@ceph.osd.2.smithi087.stdout:Apr 02 15:41:57 smithi087 podman[66133]: 2024-04-02 15:41:57.045138581 +0000 UTC m=+0.450685898 container attach f26624e9babfea270556d82570f1734a4373ac49d98dedbc6df29b62dcf6e411 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-2-deactivate, version=8, GIT_BRANCH=HEAD, distribution-scope=public, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, name=centos-stream, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, release=754, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-02T15:41:57.537 INFO:journalctl@ceph.osd.2.smithi087.stdout:Apr 02 15:41:57 smithi087 podman[66133]: 2024-04-02 15:41:57.24742571 +0000 UTC m=+0.652973030 container died f26624e9babfea270556d82570f1734a4373ac49d98dedbc6df29b62dcf6e411 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-2-deactivate, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, name=centos-stream, io.buildah.version=1.19.8, io.openshift.expose-services=, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, distribution-scope=public, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-02T15:41:57.789 INFO:journalctl@ceph.osd.2.smithi087.stdout:Apr 02 15:41:57 smithi087 podman[66159]: 2024-04-02 15:41:57.790102241 +0000 UTC m=+0.535320063 container remove f26624e9babfea270556d82570f1734a4373ac49d98dedbc6df29b62dcf6e411 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-2-deactivate, distribution-scope=public, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , vcs-type=git, name=centos-stream, ceph=True, architecture=x86_64, version=8, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, io.buildah.version=1.19.8) 2024-04-02T15:41:57.810 DEBUG:teuthology.orchestra.run.smithi087:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0400842c-f105-11ee-b647-cb9ed24678a4@osd.2.service' 2024-04-02T15:41:57.851 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T15:41:57.851 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-04-02T15:41:57.851 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-04-02T15:41:57.852 DEBUG:teuthology.orchestra.run.smithi087:> sudo systemctl stop ceph-0400842c-f105-11ee-b647-cb9ed24678a4@osd.3 2024-04-02T15:41:58.065 INFO:journalctl@ceph.osd.3.smithi087.stdout:Apr 02 15:41:57 smithi087 systemd[1]: Stopping Ceph osd.3 for 0400842c-f105-11ee-b647-cb9ed24678a4... 2024-04-02T15:41:58.066 INFO:journalctl@ceph.osd.3.smithi087.stdout:Apr 02 15:41:57 smithi087 bash[66213]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd.3" found: no such container 2024-04-02T15:41:58.344 INFO:journalctl@ceph.osd.3.smithi087.stdout:Apr 02 15:41:58 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-3[41873]: 2024-04-02T15:41:58.067+0000 7f0a73f4e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T15:41:58.344 INFO:journalctl@ceph.osd.3.smithi087.stdout:Apr 02 15:41:58 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-3[41873]: 2024-04-02T15:41:58.067+0000 7f0a73f4e700 -1 osd.3 96 *** Got signal Terminated *** 2024-04-02T15:41:58.344 INFO:journalctl@ceph.osd.3.smithi087.stdout:Apr 02 15:41:58 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-3[41873]: 2024-04-02T15:41:58.067+0000 7f0a73f4e700 -1 osd.3 96 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T15:42:01.777 INFO:journalctl@ceph.osd.3.smithi087.stdout:Apr 02 15:42:01 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-3[41873]: 2024-04-02T15:42:01.487+0000 7f0a6f0e6700 -1 osd.3 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:37.951591+0000 front 2024-04-02T15:41:37.951698+0000 (oldest deadline 2024-04-02T15:42:01.352149+0000) 2024-04-02T15:42:01.777 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:42:01 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:42:01.640Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": dial tcp 172.21.15.87:8443: connect: connection refused" 2024-04-02T15:42:01.777 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:42:01 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:42:01.640Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": dial tcp 172.21.15.87:8443: connect: connection refused" 2024-04-02T15:42:01.777 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:42:01 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:42:01.640Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:42:01.868 INFO:journalctl@ceph.osd.4.smithi106.stdout:Apr 02 15:42:01 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-4[30558]: 2024-04-02T15:42:01.535+0000 7fb659c62700 -1 osd.4 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:38.594288+0000 front 2024-04-02T15:41:38.594228+0000 (oldest deadline 2024-04-02T15:42:00.894160+0000) 2024-04-02T15:42:02.867 INFO:journalctl@ceph.osd.4.smithi106.stdout:Apr 02 15:42:02 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-4[30558]: 2024-04-02T15:42:02.581+0000 7fb659c62700 -1 osd.4 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:38.594288+0000 front 2024-04-02T15:41:38.594228+0000 (oldest deadline 2024-04-02T15:42:00.894160+0000) 2024-04-02T15:42:02.871 INFO:journalctl@ceph.osd.3.smithi087.stdout:Apr 02 15:42:02 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-3[41873]: 2024-04-02T15:42:02.524+0000 7f0a6f0e6700 -1 osd.3 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:37.951591+0000 front 2024-04-02T15:41:37.951698+0000 (oldest deadline 2024-04-02T15:42:01.352149+0000) 2024-04-02T15:42:03.531 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:03 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:03.268+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:37.624461+0000 front 2024-04-02T15:41:37.624397+0000 (oldest deadline 2024-04-02T15:42:02.324413+0000) 2024-04-02T15:42:03.788 INFO:journalctl@ceph.osd.3.smithi087.stdout:Apr 02 15:42:03 smithi087 podman[66224]: 2024-04-02 15:42:03.527351004 +0000 UTC m=+5.560492179 container died 00c03ac366b2878c271984fa764f88c98802f0c75351ef8f47d1883e8c6a4fd3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-3, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, name=centos-stream, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, RELEASE=HEAD, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, release=754, version=8, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, io.k8s.display-name=CentOS Stream 8) 2024-04-02T15:42:03.788 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:42:03 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:42:03.532Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": dial tcp 172.21.15.87:8443: connect: connection refused" 2024-04-02T15:42:03.788 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:42:03 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:42:03.533Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": dial tcp 172.21.15.87:8443: connect: connection refused" 2024-04-02T15:42:03.788 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:42:03 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:42:03.533Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:42:03.867 INFO:journalctl@ceph.osd.4.smithi106.stdout:Apr 02 15:42:03 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-4[30558]: 2024-04-02T15:42:03.589+0000 7fb659c62700 -1 osd.4 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:38.594288+0000 front 2024-04-02T15:41:38.594228+0000 (oldest deadline 2024-04-02T15:42:00.894160+0000) 2024-04-02T15:42:03.868 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:03 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:03.534+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:39.131994+0000 front 2024-04-02T15:41:39.132026+0000 (oldest deadline 2024-04-02T15:42:03.231975+0000) 2024-04-02T15:42:04.067 INFO:journalctl@ceph.osd.3.smithi087.stdout:Apr 02 15:42:03 smithi087 podman[66224]: 2024-04-02 15:42:03.86477534 +0000 UTC m=+5.897916513 container cleanup 00c03ac366b2878c271984fa764f88c98802f0c75351ef8f47d1883e8c6a4fd3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-3, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, version=8, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, architecture=x86_64, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, release=754, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-02T15:42:04.068 INFO:journalctl@ceph.osd.3.smithi087.stdout:Apr 02 15:42:03 smithi087 bash[66224]: ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-3 2024-04-02T15:42:04.068 INFO:journalctl@ceph.osd.3.smithi087.stdout:Apr 02 15:42:04 smithi087 podman[66352]: 2024-04-02 15:42:04.032542687 +0000 UTC m=+0.508791056 container remove 00c03ac366b2878c271984fa764f88c98802f0c75351ef8f47d1883e8c6a4fd3 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-3, io.openshift.tags=base centos centos-stream, release=754, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, version=8, GIT_CLEAN=True, com.redhat.component=centos-stream-container) 2024-04-02T15:42:04.068 INFO:journalctl@ceph.osd.3.smithi087.stdout:Apr 02 15:42:04 smithi087 bash[66364]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd.3" found: no such container 2024-04-02T15:42:04.371 INFO:journalctl@ceph.osd.3.smithi087.stdout:Apr 02 15:42:04 smithi087 podman[66432]: 2024-04-02 15:42:04.209257644 +0000 UTC m=+0.018087115 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-02T15:42:04.372 INFO:journalctl@ceph.osd.3.smithi087.stdout:Apr 02 15:42:04 smithi087 podman[66432]: 2024-04-02 15:42:04.316702022 +0000 UTC m=+0.125531493 container create 46e120fb2232e231e3ebe583692328091616b32be987bf9fe751ae7898e4e281 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-3-deactivate, vcs-type=git, architecture=x86_64, GIT_CLEAN=True, ceph=True, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, version=8, io.openshift.expose-services=, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , release=754, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, name=centos-stream, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-02T15:42:04.543 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:04 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5[33427]: 2024-04-02T15:42:04.514+0000 7fe314d82700 -1 osd.5 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:38.610486+0000 front 2024-04-02T15:41:38.610458+0000 (oldest deadline 2024-04-02T15:42:03.810900+0000) 2024-04-02T15:42:04.543 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:04 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:04.295+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:37.624461+0000 front 2024-04-02T15:41:37.624397+0000 (oldest deadline 2024-04-02T15:42:02.324413+0000) 2024-04-02T15:42:04.810 INFO:journalctl@ceph.osd.3.smithi087.stdout:Apr 02 15:42:04 smithi087 podman[66432]: 2024-04-02 15:42:04.527564664 +0000 UTC m=+0.336394149 container init 46e120fb2232e231e3ebe583692328091616b32be987bf9fe751ae7898e4e281 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-3-deactivate, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, release=754, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, io.openshift.tags=base centos centos-stream, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870) 2024-04-02T15:42:04.810 INFO:journalctl@ceph.osd.3.smithi087.stdout:Apr 02 15:42:04 smithi087 podman[66432]: 2024-04-02 15:42:04.530616976 +0000 UTC m=+0.339446447 container start 46e120fb2232e231e3ebe583692328091616b32be987bf9fe751ae7898e4e281 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-3-deactivate, maintainer=Guillaume Abrioux , name=centos-stream, vcs-type=git, io.buildah.version=1.19.8, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, GIT_BRANCH=HEAD, io.openshift.expose-services=, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, release=754) 2024-04-02T15:42:04.810 INFO:journalctl@ceph.osd.3.smithi087.stdout:Apr 02 15:42:04 smithi087 podman[66432]: 2024-04-02 15:42:04.610985283 +0000 UTC m=+0.419814777 container attach 46e120fb2232e231e3ebe583692328091616b32be987bf9fe751ae7898e4e281 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-3-deactivate, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, ceph=True, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., architecture=x86_64, name=centos-stream, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, vcs-type=git, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8) 2024-04-02T15:42:04.810 INFO:journalctl@ceph.osd.3.smithi087.stdout:Apr 02 15:42:04 smithi087 podman[66432]: 2024-04-02 15:42:04.81262776 +0000 UTC m=+0.621457236 container died 46e120fb2232e231e3ebe583692328091616b32be987bf9fe751ae7898e4e281 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-3-deactivate, io.k8s.display-name=CentOS Stream 8, ceph=True, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, version=8, vcs-type=git, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64) 2024-04-02T15:42:04.867 INFO:journalctl@ceph.osd.4.smithi106.stdout:Apr 02 15:42:04 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-4[30558]: 2024-04-02T15:42:04.629+0000 7fb659c62700 -1 osd.4 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:38.594288+0000 front 2024-04-02T15:41:38.594228+0000 (oldest deadline 2024-04-02T15:42:00.894160+0000) 2024-04-02T15:42:04.868 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:04 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:04.544+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:39.131994+0000 front 2024-04-02T15:41:39.132026+0000 (oldest deadline 2024-04-02T15:42:03.231975+0000) 2024-04-02T15:42:05.351 INFO:journalctl@ceph.osd.3.smithi087.stdout:Apr 02 15:42:05 smithi087 podman[66458]: 2024-04-02 15:42:05.330916084 +0000 UTC m=+0.512391998 container remove 46e120fb2232e231e3ebe583692328091616b32be987bf9fe751ae7898e4e281 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-3-deactivate, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, vcs-type=git, RELEASE=HEAD, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, version=8, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870) 2024-04-02T15:42:05.356 DEBUG:teuthology.orchestra.run.smithi087:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0400842c-f105-11ee-b647-cb9ed24678a4@osd.3.service' 2024-04-02T15:42:05.397 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T15:42:05.397 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-04-02T15:42:05.397 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-04-02T15:42:05.397 DEBUG:teuthology.orchestra.run.smithi106:> sudo systemctl stop ceph-0400842c-f105-11ee-b647-cb9ed24678a4@osd.4 2024-04-02T15:42:05.618 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:05 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5[33427]: 2024-04-02T15:42:05.497+0000 7fe314d82700 -1 osd.5 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:38.610486+0000 front 2024-04-02T15:41:38.610458+0000 (oldest deadline 2024-04-02T15:42:03.810900+0000) 2024-04-02T15:42:05.618 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:05 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:05.341+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:37.624461+0000 front 2024-04-02T15:41:37.624397+0000 (oldest deadline 2024-04-02T15:42:02.324413+0000) 2024-04-02T15:42:05.618 INFO:journalctl@ceph.osd.4.smithi106.stdout:Apr 02 15:42:05 smithi106 systemd[1]: Stopping Ceph osd.4 for 0400842c-f105-11ee-b647-cb9ed24678a4... 2024-04-02T15:42:05.619 INFO:journalctl@ceph.osd.4.smithi106.stdout:Apr 02 15:42:05 smithi106 bash[55298]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd.4" found: no such container 2024-04-02T15:42:05.619 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:05 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:05.580+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:39.131994+0000 front 2024-04-02T15:41:39.132026+0000 (oldest deadline 2024-04-02T15:42:03.231975+0000) 2024-04-02T15:42:06.117 INFO:journalctl@ceph.osd.4.smithi106.stdout:Apr 02 15:42:05 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-4[30558]: 2024-04-02T15:42:05.645+0000 7fb659c62700 -1 osd.4 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:38.594288+0000 front 2024-04-02T15:41:38.594228+0000 (oldest deadline 2024-04-02T15:42:00.894160+0000) 2024-04-02T15:42:06.118 INFO:journalctl@ceph.osd.4.smithi106.stdout:Apr 02 15:42:05 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-4[30558]: 2024-04-02T15:42:05.651+0000 7fb65cac6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T15:42:06.118 INFO:journalctl@ceph.osd.4.smithi106.stdout:Apr 02 15:42:05 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-4[30558]: 2024-04-02T15:42:05.651+0000 7fb65cac6700 -1 osd.4 96 *** Got signal Terminated *** 2024-04-02T15:42:06.118 INFO:journalctl@ceph.osd.4.smithi106.stdout:Apr 02 15:42:05 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-4[30558]: 2024-04-02T15:42:05.651+0000 7fb65cac6700 -1 osd.4 96 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T15:42:06.617 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:06 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5[33427]: 2024-04-02T15:42:06.480+0000 7fe314d82700 -1 osd.5 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:38.610486+0000 front 2024-04-02T15:41:38.610458+0000 (oldest deadline 2024-04-02T15:42:03.810900+0000) 2024-04-02T15:42:06.618 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:06 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:06.304+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:37.624461+0000 front 2024-04-02T15:41:37.624397+0000 (oldest deadline 2024-04-02T15:42:02.324413+0000) 2024-04-02T15:42:07.117 INFO:journalctl@ceph.osd.4.smithi106.stdout:Apr 02 15:42:06 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-4[30558]: 2024-04-02T15:42:06.636+0000 7fb659c62700 -1 osd.4 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:38.594288+0000 front 2024-04-02T15:41:38.594228+0000 (oldest deadline 2024-04-02T15:42:00.894160+0000) 2024-04-02T15:42:07.118 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:06 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:06.629+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:39.131994+0000 front 2024-04-02T15:41:39.132026+0000 (oldest deadline 2024-04-02T15:42:03.231975+0000) 2024-04-02T15:42:07.584 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:07 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5[33427]: 2024-04-02T15:42:07.518+0000 7fe314d82700 -1 osd.5 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:38.610486+0000 front 2024-04-02T15:41:38.610458+0000 (oldest deadline 2024-04-02T15:42:03.810900+0000) 2024-04-02T15:42:07.584 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:07 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:07.279+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:37.624461+0000 front 2024-04-02T15:41:37.624397+0000 (oldest deadline 2024-04-02T15:42:02.324413+0000) 2024-04-02T15:42:07.867 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:07 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:07.585+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:39.131994+0000 front 2024-04-02T15:41:39.132026+0000 (oldest deadline 2024-04-02T15:42:03.231975+0000) 2024-04-02T15:42:07.868 INFO:journalctl@ceph.osd.4.smithi106.stdout:Apr 02 15:42:07 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-4[30558]: 2024-04-02T15:42:07.680+0000 7fb659c62700 -1 osd.4 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:38.594288+0000 front 2024-04-02T15:41:38.594228+0000 (oldest deadline 2024-04-02T15:42:00.894160+0000) 2024-04-02T15:42:08.554 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:08.254+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:37.624461+0000 front 2024-04-02T15:41:37.624397+0000 (oldest deadline 2024-04-02T15:42:02.324413+0000) 2024-04-02T15:42:08.867 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5[33427]: 2024-04-02T15:42:08.555+0000 7fe314d82700 -1 osd.5 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:38.610486+0000 front 2024-04-02T15:41:38.610458+0000 (oldest deadline 2024-04-02T15:42:03.810900+0000) 2024-04-02T15:42:08.868 INFO:journalctl@ceph.osd.4.smithi106.stdout:Apr 02 15:42:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-4[30558]: 2024-04-02T15:42:08.667+0000 7fb659c62700 -1 osd.4 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:38.594288+0000 front 2024-04-02T15:41:38.594228+0000 (oldest deadline 2024-04-02T15:42:00.894160+0000) 2024-04-02T15:42:08.868 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:08 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:08.580+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:39.131994+0000 front 2024-04-02T15:41:39.132026+0000 (oldest deadline 2024-04-02T15:42:03.231975+0000) 2024-04-02T15:42:09.571 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:09 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:09.265+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:37.624461+0000 front 2024-04-02T15:41:37.624397+0000 (oldest deadline 2024-04-02T15:42:02.324413+0000) 2024-04-02T15:42:09.867 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:09 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5[33427]: 2024-04-02T15:42:09.573+0000 7fe314d82700 -1 osd.5 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:38.610486+0000 front 2024-04-02T15:41:38.610458+0000 (oldest deadline 2024-04-02T15:42:03.810900+0000) 2024-04-02T15:42:09.868 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:09 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:09.609+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:39.131994+0000 front 2024-04-02T15:41:39.132026+0000 (oldest deadline 2024-04-02T15:42:03.231975+0000) 2024-04-02T15:42:09.868 INFO:journalctl@ceph.osd.4.smithi106.stdout:Apr 02 15:42:09 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-4[30558]: 2024-04-02T15:42:09.632+0000 7fb659c62700 -1 osd.4 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:38.594288+0000 front 2024-04-02T15:41:38.594228+0000 (oldest deadline 2024-04-02T15:42:00.894160+0000) 2024-04-02T15:42:09.868 INFO:journalctl@ceph.osd.4.smithi106.stdout:Apr 02 15:42:09 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-4[30558]: 2024-04-02T15:42:09.632+0000 7fb659c62700 -1 osd.4 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:45.595252+0000 front 2024-04-02T15:41:45.595235+0000 (oldest deadline 2024-04-02T15:42:09.095104+0000) 2024-04-02T15:42:10.570 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:10 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:10.276+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:37.624461+0000 front 2024-04-02T15:41:37.624397+0000 (oldest deadline 2024-04-02T15:42:02.324413+0000) 2024-04-02T15:42:10.570 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:10 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:10.276+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:44.625308+0000 front 2024-04-02T15:41:44.625271+0000 (oldest deadline 2024-04-02T15:42:09.925239+0000) 2024-04-02T15:42:10.867 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:10 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:10.572+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:39.131994+0000 front 2024-04-02T15:41:39.132026+0000 (oldest deadline 2024-04-02T15:42:03.231975+0000) 2024-04-02T15:42:10.868 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:10 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5[33427]: 2024-04-02T15:42:10.584+0000 7fe314d82700 -1 osd.5 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:38.610486+0000 front 2024-04-02T15:41:38.610458+0000 (oldest deadline 2024-04-02T15:42:03.810900+0000) 2024-04-02T15:42:10.868 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:10 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5[33427]: 2024-04-02T15:42:10.584+0000 7fe314d82700 -1 osd.5 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:43.811405+0000 front 2024-04-02T15:41:43.811334+0000 (oldest deadline 2024-04-02T15:42:09.711322+0000) 2024-04-02T15:42:10.868 INFO:journalctl@ceph.osd.4.smithi106.stdout:Apr 02 15:42:10 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-4[30558]: 2024-04-02T15:42:10.597+0000 7fb659c62700 -1 osd.4 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:38.594288+0000 front 2024-04-02T15:41:38.594228+0000 (oldest deadline 2024-04-02T15:42:00.894160+0000) 2024-04-02T15:42:10.868 INFO:journalctl@ceph.osd.4.smithi106.stdout:Apr 02 15:42:10 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-4[30558]: 2024-04-02T15:42:10.597+0000 7fb659c62700 -1 osd.4 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:45.595252+0000 front 2024-04-02T15:41:45.595235+0000 (oldest deadline 2024-04-02T15:42:09.095104+0000) 2024-04-02T15:42:11.500 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:11 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:11.258+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:37.624461+0000 front 2024-04-02T15:41:37.624397+0000 (oldest deadline 2024-04-02T15:42:02.324413+0000) 2024-04-02T15:42:11.500 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:11 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:11.258+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:44.625308+0000 front 2024-04-02T15:41:44.625271+0000 (oldest deadline 2024-04-02T15:42:09.925239+0000) 2024-04-02T15:42:11.501 INFO:journalctl@ceph.osd.4.smithi106.stdout:Apr 02 15:42:11 smithi106 podman[55309]: 2024-04-02 15:42:11.152256166 +0000 UTC m=+5.661164263 container died c96f419b11f6b6fc05c5987603a93f9506791705ed38f291752a1892726dba21 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-4, distribution-scope=public, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, architecture=x86_64, io.buildah.version=1.19.8, ceph=True, release=754, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, com.redhat.component=centos-stream-container, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-02T15:42:11.836 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:11 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5[33427]: 2024-04-02T15:42:11.631+0000 7fe314d82700 -1 osd.5 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:38.610486+0000 front 2024-04-02T15:41:38.610458+0000 (oldest deadline 2024-04-02T15:42:03.810900+0000) 2024-04-02T15:42:11.836 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:11 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5[33427]: 2024-04-02T15:42:11.631+0000 7fe314d82700 -1 osd.5 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:43.811405+0000 front 2024-04-02T15:41:43.811334+0000 (oldest deadline 2024-04-02T15:42:09.711322+0000) 2024-04-02T15:42:11.837 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:11 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:11.525+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:39.131994+0000 front 2024-04-02T15:41:39.132026+0000 (oldest deadline 2024-04-02T15:42:03.231975+0000) 2024-04-02T15:42:11.837 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:11 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:11.525+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:45.533171+0000 front 2024-04-02T15:41:45.533066+0000 (oldest deadline 2024-04-02T15:42:11.433111+0000) 2024-04-02T15:42:11.837 INFO:journalctl@ceph.osd.4.smithi106.stdout:Apr 02 15:42:11 smithi106 podman[55309]: 2024-04-02 15:42:11.502223209 +0000 UTC m=+6.011131312 container cleanup c96f419b11f6b6fc05c5987603a93f9506791705ed38f291752a1892726dba21 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-4, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., version=8, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, name=centos-stream, com.redhat.component=centos-stream-container, RELEASE=HEAD, ceph=True, release=754, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-02T15:42:11.837 INFO:journalctl@ceph.osd.4.smithi106.stdout:Apr 02 15:42:11 smithi106 bash[55309]: ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-4 2024-04-02T15:42:11.837 INFO:journalctl@ceph.osd.4.smithi106.stdout:Apr 02 15:42:11 smithi106 bash[55346]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd.4" found: no such container 2024-04-02T15:42:12.066 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:42:11 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:42:11.640Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": dial tcp 172.21.15.87:8443: connect: connection refused" 2024-04-02T15:42:12.066 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:42:11 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:42:11.641Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": dial tcp 172.21.15.87:8443: connect: connection refused" 2024-04-02T15:42:12.067 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:42:11 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:42:11.641Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:42:12.514 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:12 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:12.239+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:37.624461+0000 front 2024-04-02T15:41:37.624397+0000 (oldest deadline 2024-04-02T15:42:02.324413+0000) 2024-04-02T15:42:12.515 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:12 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:12.239+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:44.625308+0000 front 2024-04-02T15:41:44.625271+0000 (oldest deadline 2024-04-02T15:42:09.925239+0000) 2024-04-02T15:42:12.515 INFO:journalctl@ceph.osd.4.smithi106.stdout:Apr 02 15:42:12 smithi106 podman[55326]: 2024-04-02 15:42:12.374575453 +0000 UTC m=+1.221100284 container remove c96f419b11f6b6fc05c5987603a93f9506791705ed38f291752a1892726dba21 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-4, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, distribution-scope=public, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, name=centos-stream, ceph=True) 2024-04-02T15:42:12.867 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:12 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5[33427]: 2024-04-02T15:42:12.654+0000 7fe314d82700 -1 osd.5 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:38.610486+0000 front 2024-04-02T15:41:38.610458+0000 (oldest deadline 2024-04-02T15:42:03.810900+0000) 2024-04-02T15:42:12.868 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:12 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5[33427]: 2024-04-02T15:42:12.654+0000 7fe314d82700 -1 osd.5 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:43.811405+0000 front 2024-04-02T15:41:43.811334+0000 (oldest deadline 2024-04-02T15:42:09.711322+0000) 2024-04-02T15:42:12.868 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:12 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:12.541+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:39.131994+0000 front 2024-04-02T15:41:39.132026+0000 (oldest deadline 2024-04-02T15:42:03.231975+0000) 2024-04-02T15:42:12.868 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:12 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:12.541+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:45.533171+0000 front 2024-04-02T15:41:45.533066+0000 (oldest deadline 2024-04-02T15:42:11.433111+0000) 2024-04-02T15:42:13.211 INFO:journalctl@ceph.osd.4.smithi106.stdout:Apr 02 15:42:12 smithi106 podman[55439]: 2024-04-02 15:42:12.919279296 +0000 UTC m=+0.111086859 container create c8218420a2c0a2e2b87794a84878a1d55da8f812d5b6426b250eb48b6e970e52 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-4-deactivate, ceph=True, name=centos-stream, distribution-scope=public, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-02T15:42:13.211 INFO:journalctl@ceph.osd.4.smithi106.stdout:Apr 02 15:42:12 smithi106 podman[55439]: 2024-04-02 15:42:12.827645414 +0000 UTC m=+0.019452991 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-02T15:42:13.211 INFO:journalctl@ceph.osd.4.smithi106.stdout:Apr 02 15:42:13 smithi106 podman[55439]: 2024-04-02 15:42:13.138478795 +0000 UTC m=+0.330286373 container init c8218420a2c0a2e2b87794a84878a1d55da8f812d5b6426b250eb48b6e970e52 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-4-deactivate, release=754, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., GIT_CLEAN=True, ceph=True, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, name=centos-stream, architecture=x86_64, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-02T15:42:13.211 INFO:journalctl@ceph.osd.4.smithi106.stdout:Apr 02 15:42:13 smithi106 podman[55439]: 2024-04-02 15:42:13.141753469 +0000 UTC m=+0.333561033 container start c8218420a2c0a2e2b87794a84878a1d55da8f812d5b6426b250eb48b6e970e52 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-4-deactivate, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, architecture=x86_64, RELEASE=HEAD, io.openshift.expose-services=, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, vendor=Red Hat, Inc., ceph=True, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-02T15:42:13.212 INFO:journalctl@ceph.osd.4.smithi106.stdout:Apr 02 15:42:13 smithi106 podman[55439]: 2024-04-02 15:42:13.213226929 +0000 UTC m=+0.405034505 container attach c8218420a2c0a2e2b87794a84878a1d55da8f812d5b6426b250eb48b6e970e52 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-4-deactivate, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, release=754, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, distribution-scope=public, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, io.openshift.expose-services=, version=8) 2024-04-02T15:42:13.538 INFO:journalctl@ceph.osd.4.smithi106.stdout:Apr 02 15:42:13 smithi106 podman[55439]: 2024-04-02 15:42:13.433108357 +0000 UTC m=+0.624915922 container died c8218420a2c0a2e2b87794a84878a1d55da8f812d5b6426b250eb48b6e970e52 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-4-deactivate, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, io.openshift.expose-services=, RELEASE=HEAD, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, vendor=Red Hat, Inc., GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-02T15:42:13.539 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:13 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:13.539+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:39.131994+0000 front 2024-04-02T15:41:39.132026+0000 (oldest deadline 2024-04-02T15:42:03.231975+0000) 2024-04-02T15:42:13.539 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:13 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:13.539+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:45.533171+0000 front 2024-04-02T15:41:45.533066+0000 (oldest deadline 2024-04-02T15:42:11.433111+0000) 2024-04-02T15:42:13.539 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:13 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:13.220+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:37.624461+0000 front 2024-04-02T15:41:37.624397+0000 (oldest deadline 2024-04-02T15:42:02.324413+0000) 2024-04-02T15:42:13.540 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:13 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:13.220+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:44.625308+0000 front 2024-04-02T15:41:44.625271+0000 (oldest deadline 2024-04-02T15:42:09.925239+0000) 2024-04-02T15:42:13.789 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:13 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5[33427]: 2024-04-02T15:42:13.692+0000 7fe314d82700 -1 osd.5 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:38.610486+0000 front 2024-04-02T15:41:38.610458+0000 (oldest deadline 2024-04-02T15:42:03.810900+0000) 2024-04-02T15:42:13.790 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:13 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5[33427]: 2024-04-02T15:42:13.692+0000 7fe314d82700 -1 osd.5 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:43.811405+0000 front 2024-04-02T15:41:43.811334+0000 (oldest deadline 2024-04-02T15:42:09.711322+0000) 2024-04-02T15:42:13.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:42:13 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:42:13.533Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": dial tcp 172.21.15.87:8443: connect: connection refused" 2024-04-02T15:42:13.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:42:13 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:42:13.533Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": dial tcp 172.21.15.87:8443: connect: connection refused" 2024-04-02T15:42:13.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:42:13 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:42:13.533Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:42:14.070 DEBUG:teuthology.orchestra.run.smithi106:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0400842c-f105-11ee-b647-cb9ed24678a4@osd.4.service' 2024-04-02T15:42:14.112 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T15:42:14.112 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-04-02T15:42:14.113 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-04-02T15:42:14.113 DEBUG:teuthology.orchestra.run.smithi106:> sudo systemctl stop ceph-0400842c-f105-11ee-b647-cb9ed24678a4@osd.5 2024-04-02T15:42:14.335 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:14 smithi106 systemd[1]: Stopping Ceph osd.5 for 0400842c-f105-11ee-b647-cb9ed24678a4... 2024-04-02T15:42:14.335 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:14 smithi106 bash[55634]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd.5" found: no such container 2024-04-02T15:42:14.336 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:14 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:14.193+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:37.624461+0000 front 2024-04-02T15:41:37.624397+0000 (oldest deadline 2024-04-02T15:42:02.324413+0000) 2024-04-02T15:42:14.336 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:14 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:14.193+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:44.625308+0000 front 2024-04-02T15:41:44.625271+0000 (oldest deadline 2024-04-02T15:42:09.925239+0000) 2024-04-02T15:42:14.617 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:14 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:14.505+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:39.131994+0000 front 2024-04-02T15:41:39.132026+0000 (oldest deadline 2024-04-02T15:42:03.231975+0000) 2024-04-02T15:42:14.617 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:14 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:14.505+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:45.533171+0000 front 2024-04-02T15:41:45.533066+0000 (oldest deadline 2024-04-02T15:42:11.433111+0000) 2024-04-02T15:42:14.618 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:14 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5[33427]: 2024-04-02T15:42:14.336+0000 7fe3173e5700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T15:42:14.618 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:14 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5[33427]: 2024-04-02T15:42:14.336+0000 7fe3173e5700 -1 osd.5 96 *** Got signal Terminated *** 2024-04-02T15:42:14.618 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:14 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5[33427]: 2024-04-02T15:42:14.336+0000 7fe3173e5700 -1 osd.5 96 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T15:42:15.117 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:14 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5[33427]: 2024-04-02T15:42:14.692+0000 7fe314d82700 -1 osd.5 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:38.610486+0000 front 2024-04-02T15:41:38.610458+0000 (oldest deadline 2024-04-02T15:42:03.810900+0000) 2024-04-02T15:42:15.117 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:14 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5[33427]: 2024-04-02T15:42:14.692+0000 7fe314d82700 -1 osd.5 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:43.811405+0000 front 2024-04-02T15:41:43.811334+0000 (oldest deadline 2024-04-02T15:42:09.711322+0000) 2024-04-02T15:42:15.512 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:15 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:15.215+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:37.624461+0000 front 2024-04-02T15:41:37.624397+0000 (oldest deadline 2024-04-02T15:42:02.324413+0000) 2024-04-02T15:42:15.512 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:15 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:15.215+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:44.625308+0000 front 2024-04-02T15:41:44.625271+0000 (oldest deadline 2024-04-02T15:42:09.925239+0000) 2024-04-02T15:42:15.786 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:15 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5[33427]: 2024-04-02T15:42:15.740+0000 7fe314d82700 -1 osd.5 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:38.610486+0000 front 2024-04-02T15:41:38.610458+0000 (oldest deadline 2024-04-02T15:42:03.810900+0000) 2024-04-02T15:42:15.786 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:15 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5[33427]: 2024-04-02T15:42:15.740+0000 7fe314d82700 -1 osd.5 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:43.811405+0000 front 2024-04-02T15:41:43.811334+0000 (oldest deadline 2024-04-02T15:42:09.711322+0000) 2024-04-02T15:42:15.786 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:15 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:15.513+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:39.131994+0000 front 2024-04-02T15:41:39.132026+0000 (oldest deadline 2024-04-02T15:42:03.231975+0000) 2024-04-02T15:42:15.787 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:15 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:15.513+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:45.533171+0000 front 2024-04-02T15:41:45.533066+0000 (oldest deadline 2024-04-02T15:42:11.433111+0000) 2024-04-02T15:42:16.488 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:16 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:16.206+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:37.624461+0000 front 2024-04-02T15:41:37.624397+0000 (oldest deadline 2024-04-02T15:42:02.324413+0000) 2024-04-02T15:42:16.488 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:16 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:16.206+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:44.625308+0000 front 2024-04-02T15:41:44.625271+0000 (oldest deadline 2024-04-02T15:42:09.925239+0000) 2024-04-02T15:42:16.757 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:16 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:16.512+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:39.131994+0000 front 2024-04-02T15:41:39.132026+0000 (oldest deadline 2024-04-02T15:42:03.231975+0000) 2024-04-02T15:42:16.757 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:16 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:16.512+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:45.533171+0000 front 2024-04-02T15:41:45.533066+0000 (oldest deadline 2024-04-02T15:42:11.433111+0000) 2024-04-02T15:42:17.025 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:16 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5[33427]: 2024-04-02T15:42:16.758+0000 7fe314d82700 -1 osd.5 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:38.610486+0000 front 2024-04-02T15:41:38.610458+0000 (oldest deadline 2024-04-02T15:42:03.810900+0000) 2024-04-02T15:42:17.025 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:16 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5[33427]: 2024-04-02T15:42:16.758+0000 7fe314d82700 -1 osd.5 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:43.811405+0000 front 2024-04-02T15:41:43.811334+0000 (oldest deadline 2024-04-02T15:42:09.711322+0000) 2024-04-02T15:42:17.026 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:16 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5[33427]: 2024-04-02T15:42:16.758+0000 7fe314d82700 -1 osd.5 96 heartbeat_check: no reply from 172.21.15.87:6822 osd.2 since back 2024-04-02T15:41:54.912709+0000 front 2024-04-02T15:41:54.912718+0000 (oldest deadline 2024-04-02T15:42:16.012724+0000) 2024-04-02T15:42:17.367 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:17 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:17.204+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:37.624461+0000 front 2024-04-02T15:41:37.624397+0000 (oldest deadline 2024-04-02T15:42:02.324413+0000) 2024-04-02T15:42:17.367 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:17 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:17.204+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:44.625308+0000 front 2024-04-02T15:41:44.625271+0000 (oldest deadline 2024-04-02T15:42:09.925239+0000) 2024-04-02T15:42:17.367 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:17 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:17.204+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6822 osd.2 since back 2024-04-02T15:41:54.026134+0000 front 2024-04-02T15:41:54.026249+0000 (oldest deadline 2024-04-02T15:42:16.326090+0000) 2024-04-02T15:42:17.753 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:17 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:17.503+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:39.131994+0000 front 2024-04-02T15:41:39.132026+0000 (oldest deadline 2024-04-02T15:42:03.231975+0000) 2024-04-02T15:42:17.753 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:17 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:17.503+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:45.533171+0000 front 2024-04-02T15:41:45.533066+0000 (oldest deadline 2024-04-02T15:42:11.433111+0000) 2024-04-02T15:42:17.753 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:17 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:17.503+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6822 osd.2 since back 2024-04-02T15:41:51.433676+0000 front 2024-04-02T15:41:51.433644+0000 (oldest deadline 2024-04-02T15:42:16.733620+0000) 2024-04-02T15:42:18.060 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:17 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5[33427]: 2024-04-02T15:42:17.807+0000 7fe314d82700 -1 osd.5 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:38.610486+0000 front 2024-04-02T15:41:38.610458+0000 (oldest deadline 2024-04-02T15:42:03.810900+0000) 2024-04-02T15:42:18.060 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:17 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5[33427]: 2024-04-02T15:42:17.807+0000 7fe314d82700 -1 osd.5 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:43.811405+0000 front 2024-04-02T15:41:43.811334+0000 (oldest deadline 2024-04-02T15:42:09.711322+0000) 2024-04-02T15:42:18.060 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:17 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5[33427]: 2024-04-02T15:42:17.807+0000 7fe314d82700 -1 osd.5 96 heartbeat_check: no reply from 172.21.15.87:6822 osd.2 since back 2024-04-02T15:41:54.912709+0000 front 2024-04-02T15:41:54.912718+0000 (oldest deadline 2024-04-02T15:42:16.012724+0000) 2024-04-02T15:42:18.367 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:18 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:18.189+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:37.624461+0000 front 2024-04-02T15:41:37.624397+0000 (oldest deadline 2024-04-02T15:42:02.324413+0000) 2024-04-02T15:42:18.367 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:18 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:18.189+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:44.625308+0000 front 2024-04-02T15:41:44.625271+0000 (oldest deadline 2024-04-02T15:42:09.925239+0000) 2024-04-02T15:42:18.368 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:18 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:18.189+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6822 osd.2 since back 2024-04-02T15:41:54.026134+0000 front 2024-04-02T15:41:54.026249+0000 (oldest deadline 2024-04-02T15:42:16.326090+0000) 2024-04-02T15:42:18.786 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:18 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:18.488+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:39.131994+0000 front 2024-04-02T15:41:39.132026+0000 (oldest deadline 2024-04-02T15:42:03.231975+0000) 2024-04-02T15:42:18.786 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:18 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:18.488+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:45.533171+0000 front 2024-04-02T15:41:45.533066+0000 (oldest deadline 2024-04-02T15:42:11.433111+0000) 2024-04-02T15:42:18.786 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:18 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:18.488+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6822 osd.2 since back 2024-04-02T15:41:51.433676+0000 front 2024-04-02T15:41:51.433644+0000 (oldest deadline 2024-04-02T15:42:16.733620+0000) 2024-04-02T15:42:19.117 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:18 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5[33427]: 2024-04-02T15:42:18.787+0000 7fe314d82700 -1 osd.5 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:38.610486+0000 front 2024-04-02T15:41:38.610458+0000 (oldest deadline 2024-04-02T15:42:03.810900+0000) 2024-04-02T15:42:19.118 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:18 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5[33427]: 2024-04-02T15:42:18.787+0000 7fe314d82700 -1 osd.5 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:43.811405+0000 front 2024-04-02T15:41:43.811334+0000 (oldest deadline 2024-04-02T15:42:09.711322+0000) 2024-04-02T15:42:19.118 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:18 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5[33427]: 2024-04-02T15:42:18.787+0000 7fe314d82700 -1 osd.5 96 heartbeat_check: no reply from 172.21.15.87:6822 osd.2 since back 2024-04-02T15:41:54.912709+0000 front 2024-04-02T15:41:54.912718+0000 (oldest deadline 2024-04-02T15:42:16.012724+0000) 2024-04-02T15:42:19.491 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:19 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:19.149+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:37.624461+0000 front 2024-04-02T15:41:37.624397+0000 (oldest deadline 2024-04-02T15:42:02.324413+0000) 2024-04-02T15:42:19.491 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:19 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:19.149+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:44.625308+0000 front 2024-04-02T15:41:44.625271+0000 (oldest deadline 2024-04-02T15:42:09.925239+0000) 2024-04-02T15:42:19.491 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:19 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:19.149+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6822 osd.2 since back 2024-04-02T15:41:54.026134+0000 front 2024-04-02T15:41:54.026249+0000 (oldest deadline 2024-04-02T15:42:16.326090+0000) 2024-04-02T15:42:19.867 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:19 smithi106 podman[55647]: 2024-04-02 15:42:19.64633255 +0000 UTC m=+5.410798662 container died 91062a17779cc6d4621217ca1d2c7d0941393b816c2dcca4d7e8dd745fc1d43e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, vcs-type=git, version=8, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, distribution-scope=public, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870) 2024-04-02T15:42:19.868 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:19 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:19.492+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:39.131994+0000 front 2024-04-02T15:41:39.132026+0000 (oldest deadline 2024-04-02T15:42:03.231975+0000) 2024-04-02T15:42:19.868 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:19 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:19.492+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:45.533171+0000 front 2024-04-02T15:41:45.533066+0000 (oldest deadline 2024-04-02T15:42:11.433111+0000) 2024-04-02T15:42:19.868 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:19 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:19.492+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6822 osd.2 since back 2024-04-02T15:41:51.433676+0000 front 2024-04-02T15:41:51.433644+0000 (oldest deadline 2024-04-02T15:42:16.733620+0000) 2024-04-02T15:42:20.180 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:20 smithi106 podman[55647]: 2024-04-02 15:42:20.017182559 +0000 UTC m=+5.781648726 container cleanup 91062a17779cc6d4621217ca1d2c7d0941393b816c2dcca4d7e8dd745fc1d43e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5, GIT_CLEAN=True, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., io.openshift.expose-services=, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, ceph=True, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-02T15:42:20.181 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:20 smithi106 bash[55647]: ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5 2024-04-02T15:42:20.181 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:20 smithi106 bash[56188]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd.5" found: no such container 2024-04-02T15:42:20.451 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:20 smithi106 podman[56173]: 2024-04-02 15:42:20.276962259 +0000 UTC m=+0.626271567 container remove 91062a17779cc6d4621217ca1d2c7d0941393b816c2dcca4d7e8dd745fc1d43e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, com.redhat.component=centos-stream-container, version=8, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, GIT_CLEAN=True, name=centos-stream, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, maintainer=Guillaume Abrioux ) 2024-04-02T15:42:20.452 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:20 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:20.181+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:37.624461+0000 front 2024-04-02T15:41:37.624397+0000 (oldest deadline 2024-04-02T15:42:02.324413+0000) 2024-04-02T15:42:20.452 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:20 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:20.181+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:44.625308+0000 front 2024-04-02T15:41:44.625271+0000 (oldest deadline 2024-04-02T15:42:09.925239+0000) 2024-04-02T15:42:20.452 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:20 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:20.182+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6822 osd.2 since back 2024-04-02T15:41:54.026134+0000 front 2024-04-02T15:41:54.026249+0000 (oldest deadline 2024-04-02T15:42:16.326090+0000) 2024-04-02T15:42:20.705 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:20 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:20.476+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:39.131994+0000 front 2024-04-02T15:41:39.132026+0000 (oldest deadline 2024-04-02T15:42:03.231975+0000) 2024-04-02T15:42:20.705 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:20 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:20.476+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:45.533171+0000 front 2024-04-02T15:41:45.533066+0000 (oldest deadline 2024-04-02T15:42:11.433111+0000) 2024-04-02T15:42:20.705 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:20 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:20.476+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6822 osd.2 since back 2024-04-02T15:41:51.433676+0000 front 2024-04-02T15:41:51.433644+0000 (oldest deadline 2024-04-02T15:42:16.733620+0000) 2024-04-02T15:42:20.706 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:20 smithi106 podman[56252]: 2024-04-02 15:42:20.357965804 +0000 UTC m=+0.018698286 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-02T15:42:20.706 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:20 smithi106 podman[56252]: 2024-04-02 15:42:20.486031984 +0000 UTC m=+0.146764465 container create 99cf9538f402c32dcd115f1a390abffe532785a7bf562bd6ebef8b238333eb7f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5-deactivate, version=8, io.openshift.expose-services=, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, release=754, vcs-type=git, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, architecture=x86_64, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-02T15:42:20.706 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:20 smithi106 podman[56252]: 2024-04-02 15:42:20.705120299 +0000 UTC m=+0.365852781 container init 99cf9538f402c32dcd115f1a390abffe532785a7bf562bd6ebef8b238333eb7f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, release=754, distribution-scope=public, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, version=8, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, architecture=x86_64, GIT_BRANCH=HEAD, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-02T15:42:20.986 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:20 smithi106 podman[56252]: 2024-04-02 15:42:20.708223151 +0000 UTC m=+0.368955634 container start 99cf9538f402c32dcd115f1a390abffe532785a7bf562bd6ebef8b238333eb7f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5-deactivate, name=centos-stream, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, io.openshift.expose-services=, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, vcs-type=git, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, version=8) 2024-04-02T15:42:20.986 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:20 smithi106 podman[56252]: 2024-04-02 15:42:20.788234911 +0000 UTC m=+0.448967413 container attach 99cf9538f402c32dcd115f1a390abffe532785a7bf562bd6ebef8b238333eb7f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5-deactivate, release=754, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, name=centos-stream, io.openshift.expose-services=, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-04-02T15:42:21.272 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:21 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:21.160+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:37.624461+0000 front 2024-04-02T15:41:37.624397+0000 (oldest deadline 2024-04-02T15:42:02.324413+0000) 2024-04-02T15:42:21.272 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:21 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:21.160+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:44.625308+0000 front 2024-04-02T15:41:44.625271+0000 (oldest deadline 2024-04-02T15:42:09.925239+0000) 2024-04-02T15:42:21.273 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:21 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:21.160+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6822 osd.2 since back 2024-04-02T15:41:54.026134+0000 front 2024-04-02T15:41:54.026249+0000 (oldest deadline 2024-04-02T15:42:16.326090+0000) 2024-04-02T15:42:21.273 INFO:journalctl@ceph.osd.5.smithi106.stdout:Apr 02 15:42:20 smithi106 podman[56252]: 2024-04-02 15:42:20.985558244 +0000 UTC m=+0.646290731 container died 99cf9538f402c32dcd115f1a390abffe532785a7bf562bd6ebef8b238333eb7f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-5-deactivate, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, ceph=True, name=centos-stream, GIT_BRANCH=HEAD, RELEASE=HEAD, io.openshift.expose-services=, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, version=8, io.openshift.tags=base centos centos-stream) 2024-04-02T15:42:21.614 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:21 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:21.496+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:39.131994+0000 front 2024-04-02T15:41:39.132026+0000 (oldest deadline 2024-04-02T15:42:03.231975+0000) 2024-04-02T15:42:21.615 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:21 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:21.496+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:45.533171+0000 front 2024-04-02T15:41:45.533066+0000 (oldest deadline 2024-04-02T15:42:11.433111+0000) 2024-04-02T15:42:21.615 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:21 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:21.496+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6822 osd.2 since back 2024-04-02T15:41:51.433676+0000 front 2024-04-02T15:41:51.433644+0000 (oldest deadline 2024-04-02T15:42:16.733620+0000) 2024-04-02T15:42:21.635 DEBUG:teuthology.orchestra.run.smithi106:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0400842c-f105-11ee-b647-cb9ed24678a4@osd.5.service' 2024-04-02T15:42:21.680 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T15:42:21.681 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-04-02T15:42:21.681 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-04-02T15:42:21.681 DEBUG:teuthology.orchestra.run.smithi106:> sudo systemctl stop ceph-0400842c-f105-11ee-b647-cb9ed24678a4@osd.6 2024-04-02T15:42:21.868 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:21 smithi106 systemd[1]: Stopping Ceph osd.6 for 0400842c-f105-11ee-b647-cb9ed24678a4... 2024-04-02T15:42:21.868 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:21 smithi106 bash[56332]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd.6" found: no such container 2024-04-02T15:42:21.907 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:42:21 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:42:21.642Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": dial tcp 172.21.15.87:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:42:21.907 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:42:21 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:42:21.642Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": dial tcp 172.21.15.87:8443: connect: connection refused" 2024-04-02T15:42:21.907 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:42:21 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:42:21.642Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:42:22.137 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:21 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:21.880+0000 7f2d21516700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T15:42:22.138 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:21 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:21.880+0000 7f2d21516700 -1 osd.6 96 *** Got signal Terminated *** 2024-04-02T15:42:22.138 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:21 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:21.880+0000 7f2d21516700 -1 osd.6 96 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T15:42:22.542 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:22 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:22.140+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:37.624461+0000 front 2024-04-02T15:41:37.624397+0000 (oldest deadline 2024-04-02T15:42:02.324413+0000) 2024-04-02T15:42:22.542 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:22 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:22.140+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:44.625308+0000 front 2024-04-02T15:41:44.625271+0000 (oldest deadline 2024-04-02T15:42:09.925239+0000) 2024-04-02T15:42:22.542 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:22 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:22.140+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6822 osd.2 since back 2024-04-02T15:41:54.026134+0000 front 2024-04-02T15:41:54.026249+0000 (oldest deadline 2024-04-02T15:42:16.326090+0000) 2024-04-02T15:42:22.867 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:22 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:22.543+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:39.131994+0000 front 2024-04-02T15:41:39.132026+0000 (oldest deadline 2024-04-02T15:42:03.231975+0000) 2024-04-02T15:42:22.867 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:22 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:22.543+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:45.533171+0000 front 2024-04-02T15:41:45.533066+0000 (oldest deadline 2024-04-02T15:42:11.433111+0000) 2024-04-02T15:42:22.867 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:22 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:22.543+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6822 osd.2 since back 2024-04-02T15:41:51.433676+0000 front 2024-04-02T15:41:51.433644+0000 (oldest deadline 2024-04-02T15:42:16.733620+0000) 2024-04-02T15:42:23.533 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:23 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:23.168+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:37.624461+0000 front 2024-04-02T15:41:37.624397+0000 (oldest deadline 2024-04-02T15:42:02.324413+0000) 2024-04-02T15:42:23.533 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:23 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:23.168+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:44.625308+0000 front 2024-04-02T15:41:44.625271+0000 (oldest deadline 2024-04-02T15:42:09.925239+0000) 2024-04-02T15:42:23.534 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:23 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:23.168+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6822 osd.2 since back 2024-04-02T15:41:54.026134+0000 front 2024-04-02T15:41:54.026249+0000 (oldest deadline 2024-04-02T15:42:16.326090+0000) 2024-04-02T15:42:23.621 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:42:23 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:42:23.533Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": dial tcp 172.21.15.87:8443: connect: connection refused" 2024-04-02T15:42:23.622 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:42:23 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:42:23.534Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": dial tcp 172.21.15.87:8443: connect: connection refused" 2024-04-02T15:42:23.622 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:42:23 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:42:23.534Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:42:23.867 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:23 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:23.534+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:39.131994+0000 front 2024-04-02T15:41:39.132026+0000 (oldest deadline 2024-04-02T15:42:03.231975+0000) 2024-04-02T15:42:23.868 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:23 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:23.534+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:45.533171+0000 front 2024-04-02T15:41:45.533066+0000 (oldest deadline 2024-04-02T15:42:11.433111+0000) 2024-04-02T15:42:23.868 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:23 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:23.534+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6822 osd.2 since back 2024-04-02T15:41:51.433676+0000 front 2024-04-02T15:41:51.433644+0000 (oldest deadline 2024-04-02T15:42:16.733620+0000) 2024-04-02T15:42:24.367 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:24 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:24.212+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:37.624461+0000 front 2024-04-02T15:41:37.624397+0000 (oldest deadline 2024-04-02T15:42:02.324413+0000) 2024-04-02T15:42:24.367 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:24 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:24.212+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:44.625308+0000 front 2024-04-02T15:41:44.625271+0000 (oldest deadline 2024-04-02T15:42:09.925239+0000) 2024-04-02T15:42:24.368 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:24 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:24.212+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6822 osd.2 since back 2024-04-02T15:41:54.026134+0000 front 2024-04-02T15:41:54.026249+0000 (oldest deadline 2024-04-02T15:42:16.326090+0000) 2024-04-02T15:42:24.867 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:24 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:24.544+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:39.131994+0000 front 2024-04-02T15:41:39.132026+0000 (oldest deadline 2024-04-02T15:42:03.231975+0000) 2024-04-02T15:42:24.868 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:24 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:24.544+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:45.533171+0000 front 2024-04-02T15:41:45.533066+0000 (oldest deadline 2024-04-02T15:42:11.433111+0000) 2024-04-02T15:42:24.868 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:24 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:24.544+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6822 osd.2 since back 2024-04-02T15:41:51.433676+0000 front 2024-04-02T15:41:51.433644+0000 (oldest deadline 2024-04-02T15:42:16.733620+0000) 2024-04-02T15:42:25.533 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:25 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:25.238+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:37.624461+0000 front 2024-04-02T15:41:37.624397+0000 (oldest deadline 2024-04-02T15:42:02.324413+0000) 2024-04-02T15:42:25.533 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:25 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:25.238+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:44.625308+0000 front 2024-04-02T15:41:44.625271+0000 (oldest deadline 2024-04-02T15:42:09.925239+0000) 2024-04-02T15:42:25.533 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:25 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:25.238+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6822 osd.2 since back 2024-04-02T15:41:54.026134+0000 front 2024-04-02T15:41:54.026249+0000 (oldest deadline 2024-04-02T15:42:16.326090+0000) 2024-04-02T15:42:25.867 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:25 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:25.535+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:39.131994+0000 front 2024-04-02T15:41:39.132026+0000 (oldest deadline 2024-04-02T15:42:03.231975+0000) 2024-04-02T15:42:25.868 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:25 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:25.535+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:45.533171+0000 front 2024-04-02T15:41:45.533066+0000 (oldest deadline 2024-04-02T15:42:11.433111+0000) 2024-04-02T15:42:25.868 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:25 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:25.535+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6822 osd.2 since back 2024-04-02T15:41:51.433676+0000 front 2024-04-02T15:41:51.433644+0000 (oldest deadline 2024-04-02T15:42:16.733620+0000) 2024-04-02T15:42:26.579 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:26 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:26.238+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:37.624461+0000 front 2024-04-02T15:41:37.624397+0000 (oldest deadline 2024-04-02T15:42:02.324413+0000) 2024-04-02T15:42:26.579 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:26 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:26.238+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:44.625308+0000 front 2024-04-02T15:41:44.625271+0000 (oldest deadline 2024-04-02T15:42:09.925239+0000) 2024-04-02T15:42:26.579 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:26 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:26.238+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6822 osd.2 since back 2024-04-02T15:41:54.026134+0000 front 2024-04-02T15:41:54.026249+0000 (oldest deadline 2024-04-02T15:42:16.326090+0000) 2024-04-02T15:42:26.867 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:26 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:26.580+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:39.131994+0000 front 2024-04-02T15:41:39.132026+0000 (oldest deadline 2024-04-02T15:42:03.231975+0000) 2024-04-02T15:42:26.868 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:26 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:26.580+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:45.533171+0000 front 2024-04-02T15:41:45.533066+0000 (oldest deadline 2024-04-02T15:42:11.433111+0000) 2024-04-02T15:42:26.868 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:26 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6[36345]: 2024-04-02T15:42:26.580+0000 7f2d1c6ae700 -1 osd.6 96 heartbeat_check: no reply from 172.21.15.87:6822 osd.2 since back 2024-04-02T15:41:51.433676+0000 front 2024-04-02T15:41:51.433644+0000 (oldest deadline 2024-04-02T15:42:16.733620+0000) 2024-04-02T15:42:27.532 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:27 smithi106 podman[56345]: 2024-04-02 15:42:27.242553839 +0000 UTC m=+5.480032780 container died f582842222f2591f8273773fb8631e2c184f5a6cf6581e74a7119975d33301e7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6, RELEASE=HEAD, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, release=754, GIT_BRANCH=HEAD, ceph=True, vcs-type=git, distribution-scope=public, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., architecture=x86_64, name=centos-stream) 2024-04-02T15:42:27.533 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:27 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:27.241+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:37.624461+0000 front 2024-04-02T15:41:37.624397+0000 (oldest deadline 2024-04-02T15:42:02.324413+0000) 2024-04-02T15:42:27.533 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:27 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:27.241+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:44.625308+0000 front 2024-04-02T15:41:44.625271+0000 (oldest deadline 2024-04-02T15:42:09.925239+0000) 2024-04-02T15:42:27.533 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:27 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:27.241+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6822 osd.2 since back 2024-04-02T15:41:54.026134+0000 front 2024-04-02T15:41:54.026249+0000 (oldest deadline 2024-04-02T15:42:16.326090+0000) 2024-04-02T15:42:27.533 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:27 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:27.241+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6830 osd.3 since back 2024-04-02T15:42:01.527210+0000 front 2024-04-02T15:42:01.527224+0000 (oldest deadline 2024-04-02T15:42:26.827269+0000) 2024-04-02T15:42:27.867 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:27 smithi106 podman[56345]: 2024-04-02 15:42:27.64449426 +0000 UTC m=+5.881973198 container cleanup f582842222f2591f8273773fb8631e2c184f5a6cf6581e74a7119975d33301e7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, version=8, com.redhat.component=centos-stream-container, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, maintainer=Guillaume Abrioux , name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, ceph=True) 2024-04-02T15:42:27.868 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:27 smithi106 bash[56345]: ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6 2024-04-02T15:42:27.868 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:27 smithi106 bash[56374]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd.6" found: no such container 2024-04-02T15:42:28.152 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:27 smithi106 podman[56362]: 2024-04-02 15:42:27.895569778 +0000 UTC m=+0.650166225 container remove f582842222f2591f8273773fb8631e2c184f5a6cf6581e74a7119975d33301e7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, ceph=True, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, version=8, vendor=Red Hat, Inc., release=754, vcs-type=git, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-02T15:42:28.152 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:28 smithi106 podman[56437]: 2024-04-02 15:42:28.036976084 +0000 UTC m=+0.018865685 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-02T15:42:28.152 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:28 smithi106 podman[56437]: 2024-04-02 15:42:28.154658434 +0000 UTC m=+0.136548016 container create 7b7341c89a9d1b2bda933019a10dc1ba8a748c113cb8e715826bbf9da78f6323 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6-deactivate, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, ceph=True, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.openshift.expose-services=, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , name=centos-stream, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8) 2024-04-02T15:42:28.471 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:28 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:28.273+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:37.624461+0000 front 2024-04-02T15:41:37.624397+0000 (oldest deadline 2024-04-02T15:42:02.324413+0000) 2024-04-02T15:42:28.471 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:28 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:28.273+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:44.625308+0000 front 2024-04-02T15:41:44.625271+0000 (oldest deadline 2024-04-02T15:42:09.925239+0000) 2024-04-02T15:42:28.471 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:28 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:28.273+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6822 osd.2 since back 2024-04-02T15:41:54.026134+0000 front 2024-04-02T15:41:54.026249+0000 (oldest deadline 2024-04-02T15:42:16.326090+0000) 2024-04-02T15:42:28.471 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:28 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:28.273+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6830 osd.3 since back 2024-04-02T15:42:01.527210+0000 front 2024-04-02T15:42:01.527224+0000 (oldest deadline 2024-04-02T15:42:26.827269+0000) 2024-04-02T15:42:28.472 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:28 smithi106 podman[56437]: 2024-04-02 15:42:28.390631316 +0000 UTC m=+0.372520903 container init 7b7341c89a9d1b2bda933019a10dc1ba8a748c113cb8e715826bbf9da78f6323 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6-deactivate, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, version=8, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, distribution-scope=public, GIT_BRANCH=HEAD, name=centos-stream, ceph=True, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-02T15:42:28.472 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:28 smithi106 podman[56437]: 2024-04-02 15:42:28.393843486 +0000 UTC m=+0.375733070 container start 7b7341c89a9d1b2bda933019a10dc1ba8a748c113cb8e715826bbf9da78f6323 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6-deactivate, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, architecture=x86_64, name=centos-stream, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, version=8, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, distribution-scope=public, com.redhat.component=centos-stream-container) 2024-04-02T15:42:28.472 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:28 smithi106 podman[56437]: 2024-04-02 15:42:28.473764655 +0000 UTC m=+0.455654239 container attach 7b7341c89a9d1b2bda933019a10dc1ba8a748c113cb8e715826bbf9da78f6323 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6-deactivate, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, release=754, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., name=centos-stream, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream) 2024-04-02T15:42:28.867 INFO:journalctl@ceph.osd.6.smithi106.stdout:Apr 02 15:42:28 smithi106 podman[56437]: 2024-04-02 15:42:28.668897551 +0000 UTC m=+0.650787125 container died 7b7341c89a9d1b2bda933019a10dc1ba8a748c113cb8e715826bbf9da78f6323 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-6-deactivate, distribution-scope=public, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.openshift.expose-services=, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, architecture=x86_64, release=754, version=8) 2024-04-02T15:42:29.289 DEBUG:teuthology.orchestra.run.smithi106:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0400842c-f105-11ee-b647-cb9ed24678a4@osd.6.service' 2024-04-02T15:42:29.330 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T15:42:29.330 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-04-02T15:42:29.331 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-04-02T15:42:29.331 DEBUG:teuthology.orchestra.run.smithi106:> sudo systemctl stop ceph-0400842c-f105-11ee-b647-cb9ed24678a4@osd.7 2024-04-02T15:42:29.536 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:29 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:29.242+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:37.624461+0000 front 2024-04-02T15:41:37.624397+0000 (oldest deadline 2024-04-02T15:42:02.324413+0000) 2024-04-02T15:42:29.536 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:29 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:29.242+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:44.625308+0000 front 2024-04-02T15:41:44.625271+0000 (oldest deadline 2024-04-02T15:42:09.925239+0000) 2024-04-02T15:42:29.536 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:29 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:29.242+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6822 osd.2 since back 2024-04-02T15:41:54.026134+0000 front 2024-04-02T15:41:54.026249+0000 (oldest deadline 2024-04-02T15:42:16.326090+0000) 2024-04-02T15:42:29.536 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:29 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:29.242+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6830 osd.3 since back 2024-04-02T15:42:01.527210+0000 front 2024-04-02T15:42:01.527224+0000 (oldest deadline 2024-04-02T15:42:26.827269+0000) 2024-04-02T15:42:29.536 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:29 smithi106 systemd[1]: Stopping Ceph osd.7 for 0400842c-f105-11ee-b647-cb9ed24678a4... 2024-04-02T15:42:29.536 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:29 smithi106 bash[56516]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd.7" found: no such container 2024-04-02T15:42:29.867 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:29 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:29.537+0000 7ff8ccb4d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T15:42:29.867 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:29 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:29.537+0000 7ff8ccb4d700 -1 osd.7 96 *** Got signal Terminated *** 2024-04-02T15:42:29.868 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:29 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:29.537+0000 7ff8ccb4d700 -1 osd.7 96 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-02T15:42:30.617 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:30 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:30.242+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:37.624461+0000 front 2024-04-02T15:41:37.624397+0000 (oldest deadline 2024-04-02T15:42:02.324413+0000) 2024-04-02T15:42:30.618 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:30 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:30.242+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:44.625308+0000 front 2024-04-02T15:41:44.625271+0000 (oldest deadline 2024-04-02T15:42:09.925239+0000) 2024-04-02T15:42:30.618 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:30 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:30.242+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6822 osd.2 since back 2024-04-02T15:41:54.026134+0000 front 2024-04-02T15:41:54.026249+0000 (oldest deadline 2024-04-02T15:42:16.326090+0000) 2024-04-02T15:42:30.618 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:30 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:30.242+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6830 osd.3 since back 2024-04-02T15:42:01.527210+0000 front 2024-04-02T15:42:01.527224+0000 (oldest deadline 2024-04-02T15:42:26.827269+0000) 2024-04-02T15:42:31.533 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:31 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:31.275+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:37.624461+0000 front 2024-04-02T15:41:37.624397+0000 (oldest deadline 2024-04-02T15:42:02.324413+0000) 2024-04-02T15:42:31.533 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:31 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:31.275+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:44.625308+0000 front 2024-04-02T15:41:44.625271+0000 (oldest deadline 2024-04-02T15:42:09.925239+0000) 2024-04-02T15:42:31.533 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:31 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:31.275+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6822 osd.2 since back 2024-04-02T15:41:54.026134+0000 front 2024-04-02T15:41:54.026249+0000 (oldest deadline 2024-04-02T15:42:16.326090+0000) 2024-04-02T15:42:31.533 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:31 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:31.275+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6830 osd.3 since back 2024-04-02T15:42:01.527210+0000 front 2024-04-02T15:42:01.527224+0000 (oldest deadline 2024-04-02T15:42:26.827269+0000) 2024-04-02T15:42:32.121 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:42:31 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:42:31.642Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": dial tcp 172.21.15.87:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:42:32.122 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:42:31 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:42:31.643Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": dial tcp 172.21.15.87:8443: connect: connection refused" 2024-04-02T15:42:32.122 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:42:31 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:42:31.643Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:42:32.606 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:32 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:32.302+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:37.624461+0000 front 2024-04-02T15:41:37.624397+0000 (oldest deadline 2024-04-02T15:42:02.324413+0000) 2024-04-02T15:42:32.606 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:32 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:32.302+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:44.625308+0000 front 2024-04-02T15:41:44.625271+0000 (oldest deadline 2024-04-02T15:42:09.925239+0000) 2024-04-02T15:42:32.606 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:32 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:32.302+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6822 osd.2 since back 2024-04-02T15:41:54.026134+0000 front 2024-04-02T15:41:54.026249+0000 (oldest deadline 2024-04-02T15:42:16.326090+0000) 2024-04-02T15:42:32.606 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:32 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:32.302+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6830 osd.3 since back 2024-04-02T15:42:01.527210+0000 front 2024-04-02T15:42:01.527224+0000 (oldest deadline 2024-04-02T15:42:26.827269+0000) 2024-04-02T15:42:33.617 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:33 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:33.308+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:37.624461+0000 front 2024-04-02T15:41:37.624397+0000 (oldest deadline 2024-04-02T15:42:02.324413+0000) 2024-04-02T15:42:33.618 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:33 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:33.308+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:44.625308+0000 front 2024-04-02T15:41:44.625271+0000 (oldest deadline 2024-04-02T15:42:09.925239+0000) 2024-04-02T15:42:33.618 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:33 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:33.308+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6822 osd.2 since back 2024-04-02T15:41:54.026134+0000 front 2024-04-02T15:41:54.026249+0000 (oldest deadline 2024-04-02T15:42:16.326090+0000) 2024-04-02T15:42:33.618 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:33 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:33.308+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6830 osd.3 since back 2024-04-02T15:42:01.527210+0000 front 2024-04-02T15:42:01.527224+0000 (oldest deadline 2024-04-02T15:42:26.827269+0000) 2024-04-02T15:42:33.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:42:33 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=error ts=2024-04-02T15:42:33.534Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=7 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.87:8443//api/prometheus_receiver\": dial tcp 172.21.15.87:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:42:33.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:42:33 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:42:33.535Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.87:8443//api/prometheus_receiver\": dial tcp 172.21.15.87:8443: connect: connection refused" 2024-04-02T15:42:33.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:42:33 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=warn ts=2024-04-02T15:42:33.535Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.106:8443/api/prometheus_receiver\": dial tcp 172.21.15.106:8443: connect: connection refused" 2024-04-02T15:42:34.617 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:34 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:34.338+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6806 osd.0 since back 2024-04-02T15:41:37.624461+0000 front 2024-04-02T15:41:37.624397+0000 (oldest deadline 2024-04-02T15:42:02.324413+0000) 2024-04-02T15:42:34.618 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:34 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:34.338+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6814 osd.1 since back 2024-04-02T15:41:44.625308+0000 front 2024-04-02T15:41:44.625271+0000 (oldest deadline 2024-04-02T15:42:09.925239+0000) 2024-04-02T15:42:34.618 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:34 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:34.338+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6822 osd.2 since back 2024-04-02T15:41:54.026134+0000 front 2024-04-02T15:41:54.026249+0000 (oldest deadline 2024-04-02T15:42:16.326090+0000) 2024-04-02T15:42:34.618 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:34 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7[39290]: 2024-04-02T15:42:34.338+0000 7ff8c8ce7700 -1 osd.7 96 heartbeat_check: no reply from 172.21.15.87:6830 osd.3 since back 2024-04-02T15:42:01.527210+0000 front 2024-04-02T15:42:01.527224+0000 (oldest deadline 2024-04-02T15:42:26.827269+0000) 2024-04-02T15:42:35.134 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:34 smithi106 podman[56529]: 2024-04-02 15:42:34.878190667 +0000 UTC m=+5.434921537 container died 85ed81d59bddace9d01d7d4b2eab0739a54b457f9f7272f3c40a06173e845f76 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, RELEASE=HEAD, distribution-scope=public, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, maintainer=Guillaume Abrioux , architecture=x86_64, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., GIT_CLEAN=True, vcs-type=git, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=) 2024-04-02T15:42:35.443 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:35 smithi106 podman[56529]: 2024-04-02 15:42:35.236051082 +0000 UTC m=+5.792782017 container cleanup 85ed81d59bddace9d01d7d4b2eab0739a54b457f9f7272f3c40a06173e845f76 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, RELEASE=HEAD, distribution-scope=public, vendor=Red Hat, Inc., architecture=x86_64, vcs-type=git, GIT_BRANCH=HEAD, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, CEPH_POINT_RELEASE=-17.2.0, ceph=True, io.k8s.display-name=CentOS Stream 8) 2024-04-02T15:42:35.443 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:35 smithi106 bash[56529]: ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7 2024-04-02T15:42:35.443 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:35 smithi106 bash[56626]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd.7" found: no such container 2024-04-02T15:42:35.778 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:35 smithi106 podman[56597]: 2024-04-02 15:42:35.512298618 +0000 UTC m=+0.633562506 container remove 85ed81d59bddace9d01d7d4b2eab0739a54b457f9f7272f3c40a06173e845f76 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, vcs-type=git, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, release=754, GIT_BRANCH=HEAD, ceph=True, distribution-scope=public, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-02T15:42:35.779 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:35 smithi106 podman[56770]: 2024-04-02 15:42:35.581256819 +0000 UTC m=+0.022379938 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-02T15:42:35.779 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:35 smithi106 podman[56770]: 2024-04-02 15:42:35.679711897 +0000 UTC m=+0.120835034 container create 6dc10cdd1c855add93e32d3bc8a6e3d8d77dd62de519a6711cef0840cd8174f6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, ceph=True, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, distribution-scope=public, version=8, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-02T15:42:36.117 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:35 smithi106 podman[56770]: 2024-04-02 15:42:35.8740759 +0000 UTC m=+0.315199000 container init 6dc10cdd1c855add93e32d3bc8a6e3d8d77dd62de519a6711cef0840cd8174f6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7-deactivate, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, ceph=True, RELEASE=HEAD, version=8, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-04-02T15:42:36.118 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:35 smithi106 podman[56770]: 2024-04-02 15:42:35.877553978 +0000 UTC m=+0.318677073 container start 6dc10cdd1c855add93e32d3bc8a6e3d8d77dd62de519a6711cef0840cd8174f6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7-deactivate, release=754, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, RELEASE=HEAD, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-02T15:42:36.118 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:35 smithi106 podman[56770]: 2024-04-02 15:42:35.948873134 +0000 UTC m=+0.389996244 container attach 6dc10cdd1c855add93e32d3bc8a6e3d8d77dd62de519a6711cef0840cd8174f6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7-deactivate, com.redhat.component=centos-stream-container, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, vendor=Red Hat, Inc., vcs-type=git, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True) 2024-04-02T15:42:36.591 INFO:journalctl@ceph.osd.7.smithi106.stdout:Apr 02 15:42:36 smithi106 podman[56770]: 2024-04-02 15:42:36.174480739 +0000 UTC m=+0.615603844 container died 6dc10cdd1c855add93e32d3bc8a6e3d8d77dd62de519a6711cef0840cd8174f6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-osd-7-deactivate, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, io.buildah.version=1.19.8, RELEASE=HEAD, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, ceph=True, distribution-scope=public, release=754, vendor=Red Hat, Inc., architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-02T15:42:36.610 DEBUG:teuthology.orchestra.run.smithi106:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0400842c-f105-11ee-b647-cb9ed24678a4@osd.7.service' 2024-04-02T15:42:36.656 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T15:42:36.656 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-04-02T15:42:36.657 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-04-02T15:42:36.657 DEBUG:teuthology.orchestra.run.smithi106:> sudo systemctl stop ceph-0400842c-f105-11ee-b647-cb9ed24678a4@prometheus.a 2024-04-02T15:42:36.843 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:42:36 smithi106 systemd[1]: Stopping Ceph prometheus.a for 0400842c-f105-11ee-b647-cb9ed24678a4... 2024-04-02T15:42:36.844 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:42:36 smithi106 bash[56935]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus.a" found: no such container 2024-04-02T15:42:37.117 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:42:36 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[45564]: ts=2024-04-02T15:42:36.846Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-02T15:42:37.118 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:42:36 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[45564]: ts=2024-04-02T15:42:36.846Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-02T15:42:37.118 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:42:36 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[45564]: ts=2024-04-02T15:42:36.846Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-02T15:42:37.118 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:42:36 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[45564]: ts=2024-04-02T15:42:36.846Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-02T15:42:37.118 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:42:36 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[45564]: ts=2024-04-02T15:42:36.846Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-02T15:42:37.118 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:42:36 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[45564]: ts=2024-04-02T15:42:36.846Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-02T15:42:37.118 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:42:36 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[45564]: ts=2024-04-02T15:42:36.846Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-02T15:42:37.119 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:42:36 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[45564]: ts=2024-04-02T15:42:36.846Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-02T15:42:37.119 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:42:36 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[45564]: ts=2024-04-02T15:42:36.846Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-02T15:42:37.119 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:42:36 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[45564]: ts=2024-04-02T15:42:36.846Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-02T15:42:37.119 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:42:36 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[45564]: ts=2024-04-02T15:42:36.846Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-02T15:42:37.119 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:42:36 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a[45564]: ts=2024-04-02T15:42:36.847Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-02T15:42:37.119 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:42:36 smithi106 podman[56959]: 2024-04-02 15:42:36.856688206 +0000 UTC m=+0.122374420 container died 3ea0ba4743fb84bbe082ed969e1ff6c636d24c050b93715af45c7dddcd0b5834 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T15:42:37.477 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:42:37 smithi106 podman[56959]: 2024-04-02 15:42:37.229169 +0000 UTC m=+0.494855200 container cleanup 3ea0ba4743fb84bbe082ed969e1ff6c636d24c050b93715af45c7dddcd0b5834 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-02T15:42:37.478 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:42:37 smithi106 bash[56959]: ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus-a 2024-04-02T15:42:37.478 INFO:journalctl@ceph.prometheus.a.smithi106.stdout:Apr 02 15:42:37 smithi106 bash[56998]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-prometheus.a" found: no such container 2024-04-02T15:42:37.483 DEBUG:teuthology.orchestra.run.smithi106:> sudo pkill -f 'journalctl -f -n 0 -u ceph-0400842c-f105-11ee-b647-cb9ed24678a4@prometheus.a.service' 2024-04-02T15:42:37.524 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-02T15:42:37.524 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-04-02T15:42:37.524 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 --force --keep-logs 2024-04-02T15:42:39.531 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:42:39 smithi087 systemd[1]: Stopping Ceph node-exporter.a for 0400842c-f105-11ee-b647-cb9ed24678a4... 2024-04-02T15:42:39.531 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:42:39 smithi087 bash[67694]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter.a" found: no such container 2024-04-02T15:42:39.532 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:42:39 smithi087 podman[67706]: 2024-04-02 15:42:39.277596245 +0000 UTC m=+0.104019796 container died feba1ed50125939c5960bc947f2a0933a74a9217bca82c963ab55bcc5e98798f (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-02T15:42:39.872 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:42:39 smithi087 podman[67706]: 2024-04-02 15:42:39.725176527 +0000 UTC m=+0.551600067 container cleanup feba1ed50125939c5960bc947f2a0933a74a9217bca82c963ab55bcc5e98798f (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-02T15:42:39.872 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:42:39 smithi087 bash[67706]: ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a 2024-04-02T15:42:39.872 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:42:39 smithi087 bash[67733]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter.a" found: no such container 2024-04-02T15:42:40.302 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:42:40 smithi087 systemd[1]: Stopping Ceph alertmanager.a for 0400842c-f105-11ee-b647-cb9ed24678a4... 2024-04-02T15:42:40.302 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:42:40 smithi087 bash[67785]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager.a" found: no such container 2024-04-02T15:42:40.302 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:42:40 smithi087 podman[67720]: 2024-04-02 15:42:40.001657168 +0000 UTC m=+0.727036337 container remove feba1ed50125939c5960bc947f2a0933a74a9217bca82c963ab55bcc5e98798f (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-02T15:42:40.302 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:42:40 smithi087 systemd[1]: ceph-0400842c-f105-11ee-b647-cb9ed24678a4@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-02T15:42:40.302 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:42:40 smithi087 systemd[1]: ceph-0400842c-f105-11ee-b647-cb9ed24678a4@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-02T15:42:40.303 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:42:40 smithi087 systemd[1]: Stopped Ceph node-exporter.a for 0400842c-f105-11ee-b647-cb9ed24678a4. 2024-04-02T15:42:40.303 INFO:journalctl@ceph.node-exporter.a.smithi087.stdout:Apr 02 15:42:40 smithi087 systemd[1]: ceph-0400842c-f105-11ee-b647-cb9ed24678a4@node-exporter.a.service: Consumed 5.197s CPU time. 2024-04-02T15:42:40.594 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:42:40 smithi087 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a[51534]: level=info ts=2024-04-02T15:42:40.304Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-04-02T15:42:40.594 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:42:40 smithi087 podman[67797]: 2024-04-02 15:42:40.335476802 +0000 UTC m=+0.124508865 container died 93850beda829b99b2da03b4f505a6f8d0406efd427d99369d60be5fa932fc328 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T15:42:40.871 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:42:40 smithi087 podman[67797]: 2024-04-02 15:42:40.696189881 +0000 UTC m=+0.485221928 container cleanup 93850beda829b99b2da03b4f505a6f8d0406efd427d99369d60be5fa932fc328 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T15:42:40.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:42:40 smithi087 bash[67797]: ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a 2024-04-02T15:42:40.872 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:42:40 smithi087 bash[67823]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager.a" found: no such container 2024-04-02T15:42:41.221 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:42:40 smithi087 podman[67811]: 2024-04-02 15:42:40.97211125 +0000 UTC m=+0.634970965 container remove 93850beda829b99b2da03b4f505a6f8d0406efd427d99369d60be5fa932fc328 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-02T15:42:41.221 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:42:41 smithi087 podman[67811]: 2024-04-02 15:42:41.047887226 +0000 UTC m=+0.710746929 volume remove d5294a5ebd92cd81a4bb082fd9cc5b81138b215e075c59964f01a5236da7f21d 2024-04-02T15:42:41.221 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:42:41 smithi087 systemd[1]: ceph-0400842c-f105-11ee-b647-cb9ed24678a4@alertmanager.a.service: Deactivated successfully. 2024-04-02T15:42:41.221 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:42:41 smithi087 systemd[1]: Stopped Ceph alertmanager.a for 0400842c-f105-11ee-b647-cb9ed24678a4. 2024-04-02T15:42:41.221 INFO:journalctl@ceph.alertmanager.a.smithi087.stdout:Apr 02 15:42:41 smithi087 systemd[1]: ceph-0400842c-f105-11ee-b647-cb9ed24678a4@alertmanager.a.service: Consumed 1.728s CPU time. 2024-04-02T15:43:04.128 DEBUG:teuthology.orchestra.run.smithi106:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 --force --keep-logs 2024-04-02T15:43:05.903 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:43:05 smithi106 systemd[1]: Stopping Ceph node-exporter.b for 0400842c-f105-11ee-b647-cb9ed24678a4... 2024-04-02T15:43:05.903 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:43:05 smithi106 bash[57899]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter.b" found: no such container 2024-04-02T15:43:06.220 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:43:05 smithi106 podman[57912]: 2024-04-02 15:43:05.991266031 +0000 UTC m=+0.138010652 container died fb10fb2b50d2b06c18ba92c000ab7cf8e50aa9b7704fa418f03444e9c823d934 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-02T15:43:06.579 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:43:06 smithi106 podman[57912]: 2024-04-02 15:43:06.372185035 +0000 UTC m=+0.518929724 container cleanup fb10fb2b50d2b06c18ba92c000ab7cf8e50aa9b7704fa418f03444e9c823d934 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-02T15:43:06.579 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:43:06 smithi106 bash[57912]: ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b 2024-04-02T15:43:06.579 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:43:06 smithi106 bash[57939]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter.b" found: no such container 2024-04-02T15:43:06.867 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:43:06 smithi106 systemd[1]: Stopping Ceph grafana.a for 0400842c-f105-11ee-b647-cb9ed24678a4... 2024-04-02T15:43:06.868 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:43:06 smithi106 bash[57992]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana.a" found: no such container 2024-04-02T15:43:06.868 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:43:06 smithi106 podman[57927]: 2024-04-02 15:43:06.581645594 +0000 UTC m=+0.591465321 container remove fb10fb2b50d2b06c18ba92c000ab7cf8e50aa9b7704fa418f03444e9c823d934 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-02T15:43:06.868 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:43:06 smithi106 systemd[1]: ceph-0400842c-f105-11ee-b647-cb9ed24678a4@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-02T15:43:06.868 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:43:06 smithi106 systemd[1]: ceph-0400842c-f105-11ee-b647-cb9ed24678a4@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-02T15:43:06.868 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:43:06 smithi106 systemd[1]: Stopped Ceph node-exporter.b for 0400842c-f105-11ee-b647-cb9ed24678a4. 2024-04-02T15:43:06.869 INFO:journalctl@ceph.node-exporter.b.smithi106.stdout:Apr 02 15:43:06 smithi106 systemd[1]: ceph-0400842c-f105-11ee-b647-cb9ed24678a4@node-exporter.b.service: Consumed 5.386s CPU time. 2024-04-02T15:43:07.157 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:43:06 smithi106 ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a[44561]: t=2024-04-02T15:43:06+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-04-02T15:43:07.157 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:43:06 smithi106 podman[58003]: 2024-04-02 15:43:06.894869882 +0000 UTC m=+0.097793428 container died 8956d795b0b49a3d071492d47c4e3a589ed5347bf54c4846f33b8d2bf88840d7 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, release=236.1648460182, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, architecture=x86_64, distribution-scope=public, io.buildah.version=1.24.2, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, maintainer=Paul Cuzner , vendor=Red Hat, Inc., build-date=2022-03-28T10:36:18.413762, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, io.openshift.tags=base rhel8) 2024-04-02T15:43:07.507 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:43:07 smithi106 podman[58003]: 2024-04-02 15:43:07.242583603 +0000 UTC m=+0.445507195 container cleanup 8956d795b0b49a3d071492d47c4e3a589ed5347bf54c4846f33b8d2bf88840d7 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, io.openshift.expose-services=, release=236.1648460182, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , vcs-type=git, version=8.5, architecture=x86_64, build-date=2022-03-28T10:36:18.413762, io.buildah.version=1.24.2, io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, description=Ceph Grafana Container) 2024-04-02T15:43:07.507 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:43:07 smithi106 bash[58003]: ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a 2024-04-02T15:43:07.507 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:43:07 smithi106 bash[58030]: Error: no container with name or ID "ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana.a" found: no such container 2024-04-02T15:43:07.810 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:43:07 smithi106 podman[58017]: 2024-04-02 15:43:07.509981336 +0000 UTC m=+0.615550636 container remove 8956d795b0b49a3d071492d47c4e3a589ed5347bf54c4846f33b8d2bf88840d7 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-0400842c-f105-11ee-b647-cb9ed24678a4-grafana-a, vendor=Red Hat, Inc., maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, build-date=2022-03-28T10:36:18.413762, name=ubi8, architecture=x86_64, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, io.buildah.version=1.24.2, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, release=236.1648460182, version=8.5, description=Ceph Grafana Container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-02T15:43:07.811 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:43:07 smithi106 systemd[1]: ceph-0400842c-f105-11ee-b647-cb9ed24678a4@grafana.a.service: Deactivated successfully. 2024-04-02T15:43:07.811 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:43:07 smithi106 systemd[1]: Stopped Ceph grafana.a for 0400842c-f105-11ee-b647-cb9ed24678a4. 2024-04-02T15:43:07.811 INFO:journalctl@ceph.grafana.a.smithi106.stdout:Apr 02 15:43:07 smithi106 systemd[1]: ceph-0400842c-f105-11ee-b647-cb9ed24678a4@grafana.a.service: Consumed 4.329s CPU time. 2024-04-02T15:43:18.791 DEBUG:teuthology.orchestra.run.smithi087:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-02T15:43:18.827 DEBUG:teuthology.orchestra.run.smithi106:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-02T15:43:18.858 INFO:tasks.cephadm:Archiving crash dumps... 2024-04-02T15:43:18.861 DEBUG:teuthology.misc:Transferring archived files from smithi087:/var/lib/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/crash to /home/teuthworker/archive/gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi/7636172/remote/smithi087/crash 2024-04-02T15:43:18.861 DEBUG:teuthology.orchestra.run.smithi087:> sudo tar c -f - -C /var/lib/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/crash -- . 2024-04-02T15:43:19.005 INFO:teuthology.orchestra.run.smithi087.stderr:tar: /var/lib/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/crash: Cannot open: No such file or directory 2024-04-02T15:43:19.006 INFO:teuthology.orchestra.run.smithi087.stderr:tar: Error is not recoverable: exiting now 2024-04-02T15:43:19.007 DEBUG:teuthology.misc:Transferring archived files from smithi106:/var/lib/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/crash to /home/teuthworker/archive/gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi/7636172/remote/smithi106/crash 2024-04-02T15:43:19.008 DEBUG:teuthology.orchestra.run.smithi106:> sudo tar c -f - -C /var/lib/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/crash -- . 2024-04-02T15:43:19.046 INFO:teuthology.orchestra.run.smithi106.stderr:tar: /var/lib/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/crash: Cannot open: No such file or directory 2024-04-02T15:43:19.047 INFO:teuthology.orchestra.run.smithi106.stderr:tar: Error is not recoverable: exiting now 2024-04-02T15:43:19.048 INFO:tasks.cephadm:Checking cluster log for badness... 2024-04-02T15:43:19.048 DEBUG:teuthology.orchestra.run.smithi087:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-02T15:43:19.080 INFO:teuthology.orchestra.run.smithi087.stdout:2024-04-02T15:27:04.731925+0000 mon.a (mon.0) 357 : cluster [WRN] Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-04-02T15:43:19.080 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-04-02T15:43:19.081 INFO:tasks.cephadm:Compressing logs... 2024-04-02T15:43:19.081 DEBUG:teuthology.orchestra.run.smithi087:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-02T15:43:19.123 DEBUG:teuthology.orchestra.run.smithi106:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-02T15:43:19.149 INFO:teuthology.orchestra.run.smithi087.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-02T15:43:19.149 INFO:teuthology.orchestra.run.smithi087.stderr:: No such file or directory 2024-04-02T15:43:19.149 INFO:teuthology.orchestra.run.smithi087.stderr:gzip -5 --verbose -- /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-mon.a.log 2024-04-02T15:43:19.154 INFO:teuthology.orchestra.run.smithi087.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-client.rgw.foo.smithi087.awfxzs.log 2024-04-02T15:43:19.154 INFO:teuthology.orchestra.run.smithi087.stderr:/var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph.audit.log 2024-04-02T15:43:19.154 INFO:teuthology.orchestra.run.smithi087.stderr:/var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-client.rgw.foo.smithi087.awfxzs.log: gzip -5 --verbose -- /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-osd.1.log 2024-04-02T15:43:19.154 INFO:teuthology.orchestra.run.smithi087.stderr: 62.5% -- replaced with /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-client.rgw.foo.smithi087.awfxzs.log.gz 2024-04-02T15:43:19.154 INFO:teuthology.orchestra.run.smithi087.stderr:/var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-osd.3.log 2024-04-02T15:43:19.155 INFO:teuthology.orchestra.run.smithi087.stderr:/var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-mgr.y.log 2024-04-02T15:43:19.155 INFO:teuthology.orchestra.run.smithi087.stderr:/var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/tcmu-runner.log 2024-04-02T15:43:19.155 INFO:teuthology.orchestra.run.smithi087.stderr:/var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-osd.0.log 2024-04-02T15:43:19.155 INFO:teuthology.orchestra.run.smithi087.stderr:/var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/tcmu-runner.log: gzip -5 --verbose -- /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph.cephadm.log 2024-04-02T15:43:19.155 INFO:teuthology.orchestra.run.smithi087.stderr: 66.9% -- replaced with /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/tcmu-runner.log.gz 2024-04-02T15:43:19.155 INFO:teuthology.orchestra.run.smithi087.stderr:/var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph.log 2024-04-02T15:43:19.155 INFO:teuthology.orchestra.run.smithi087.stderr:/var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph.cephadm.log: 89.3% -- replaced with /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph.cephadm.log.gz 2024-04-02T15:43:19.159 INFO:teuthology.orchestra.run.smithi087.stderr:gzip -5 --verbose -- /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-osd.2.log 2024-04-02T15:43:19.159 INFO:teuthology.orchestra.run.smithi087.stderr:/var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph.log: 94.2% -- replaced with /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph.audit.log.gz 2024-04-02T15:43:19.160 INFO:teuthology.orchestra.run.smithi087.stderr: 93.6% -- replaced with /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph.log.gz 2024-04-02T15:43:19.160 INFO:teuthology.orchestra.run.smithi087.stderr:gzip -5 --verbose -- /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-volume.log 2024-04-02T15:43:19.160 INFO:teuthology.orchestra.run.smithi087.stderr:/var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-mon.c.log 2024-04-02T15:43:19.162 INFO:teuthology.orchestra.run.smithi106.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-02T15:43:19.162 INFO:teuthology.orchestra.run.smithi106.stderr:: No such file or directory 2024-04-02T15:43:19.162 INFO:teuthology.orchestra.run.smithi106.stderr:gzip -5 --verbose -- /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-osd.6.log 2024-04-02T15:43:19.162 INFO:teuthology.orchestra.run.smithi106.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-client.rgw.foo.smithi106.bfeikp.log 2024-04-02T15:43:19.163 INFO:teuthology.orchestra.run.smithi106.stderr:/var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph.audit.log 2024-04-02T15:43:19.163 INFO:teuthology.orchestra.run.smithi106.stderr:/var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-client.rgw.foo.smithi106.bfeikp.log: gzip -5 --verbose -- /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-osd.4.log 2024-04-02T15:43:19.163 INFO:teuthology.orchestra.run.smithi106.stderr: 62.5% -- replaced with /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-client.rgw.foo.smithi106.bfeikp.log.gz 2024-04-02T15:43:19.163 INFO:teuthology.orchestra.run.smithi106.stderr:/var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-osd.7.log 2024-04-02T15:43:19.163 INFO:teuthology.orchestra.run.smithi106.stderr:gzip -5 --verbose -- /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-osd.5.log 2024-04-02T15:43:19.163 INFO:teuthology.orchestra.run.smithi106.stderr:/var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-osd.4.log: /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-osd.7.log: /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-osd.5.log: 90.6% -- replaced with /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph.audit.log.gz 2024-04-02T15:43:19.163 INFO:teuthology.orchestra.run.smithi106.stderr:gzip -5 --verbose -- /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-mgr.x.log 2024-04-02T15:43:19.163 INFO:teuthology.orchestra.run.smithi106.stderr:gzip -5 --verbose -- /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph.cephadm.log 2024-04-02T15:43:19.163 INFO:teuthology.orchestra.run.smithi106.stderr:/var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph.log 2024-04-02T15:43:19.164 INFO:teuthology.orchestra.run.smithi106.stderr:/var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph.cephadm.log: 81.8% -- replaced with /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph.cephadm.log.gz 2024-04-02T15:43:19.164 INFO:teuthology.orchestra.run.smithi106.stderr:gzip -5 --verbose -- /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-volume.log 2024-04-02T15:43:19.165 INFO:teuthology.orchestra.run.smithi106.stderr:/var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph.log: 88.6% -- replaced with /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph.log.gz 2024-04-02T15:43:19.165 INFO:teuthology.orchestra.run.smithi106.stderr:gzip -5 --verbose -- /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-mon.b.log 2024-04-02T15:43:19.172 INFO:teuthology.orchestra.run.smithi106.stderr:/var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-volume.log: 92.4% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-02T15:43:19.173 INFO:teuthology.orchestra.run.smithi087.stderr:/var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-volume.log: /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-mon.c.log: 92.2% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-02T15:43:19.182 INFO:teuthology.orchestra.run.smithi106.stderr:/var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-mon.b.log: 92.4% -- replaced with /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-mgr.x.log.gz 2024-04-02T15:43:19.190 INFO:teuthology.orchestra.run.smithi087.stderr: 92.5% -- replaced with /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-volume.log.gz 2024-04-02T15:43:19.193 INFO:teuthology.orchestra.run.smithi106.stderr: 92.6% -- replaced with /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-volume.log.gz 2024-04-02T15:43:19.379 INFO:teuthology.orchestra.run.smithi087.stderr: 87.4% -- replaced with /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-mgr.y.log.gz 2024-04-02T15:43:19.414 INFO:teuthology.orchestra.run.smithi087.stderr: 92.5% -- replaced with /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-mon.c.log.gz 2024-04-02T15:43:19.417 INFO:teuthology.orchestra.run.smithi106.stderr: 92.4% -- replaced with /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-mon.b.log.gz 2024-04-02T15:43:19.602 INFO:teuthology.orchestra.run.smithi087.stderr: 93.2% -- replaced with /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-osd.2.log.gz 2024-04-02T15:43:19.605 INFO:teuthology.orchestra.run.smithi106.stderr: 93.0% -- replaced with /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-osd.6.log.gz 2024-04-02T15:43:19.804 INFO:teuthology.orchestra.run.smithi106.stderr: 93.6% -- replaced with /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-osd.7.log.gz 2024-04-02T15:43:19.807 INFO:teuthology.orchestra.run.smithi106.stderr: 93.5% -- replaced with /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-osd.4.log.gz 2024-04-02T15:43:19.807 INFO:teuthology.orchestra.run.smithi106.stderr: 93.3% -- replaced with /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-osd.5.log.gz 2024-04-02T15:43:19.809 INFO:teuthology.orchestra.run.smithi106.stderr: 2024-04-02T15:43:19.809 INFO:teuthology.orchestra.run.smithi106.stderr:real 0m0.672s 2024-04-02T15:43:19.810 INFO:teuthology.orchestra.run.smithi106.stderr:user 0m2.648s 2024-04-02T15:43:19.810 INFO:teuthology.orchestra.run.smithi106.stderr:sys 0m0.120s 2024-04-02T15:43:19.815 INFO:teuthology.orchestra.run.smithi087.stderr: 93.3% -- replaced with /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-osd.3.log.gz 2024-04-02T15:43:19.841 INFO:teuthology.orchestra.run.smithi087.stderr: 93.4% -- replaced with /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-osd.1.log.gz 2024-04-02T15:43:19.855 INFO:teuthology.orchestra.run.smithi087.stderr: 93.3% -- replaced with /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-osd.0.log.gz 2024-04-02T15:43:19.894 INFO:teuthology.orchestra.run.smithi087.stderr: 91.0% -- replaced with /var/log/ceph/0400842c-f105-11ee-b647-cb9ed24678a4/ceph-mon.a.log.gz 2024-04-02T15:43:19.896 INFO:teuthology.orchestra.run.smithi087.stderr: 2024-04-02T15:43:19.896 INFO:teuthology.orchestra.run.smithi087.stderr:real 0m0.760s 2024-04-02T15:43:19.896 INFO:teuthology.orchestra.run.smithi087.stderr:user 0m3.636s 2024-04-02T15:43:19.896 INFO:teuthology.orchestra.run.smithi087.stderr:sys 0m0.166s 2024-04-02T15:43:19.896 INFO:tasks.cephadm:Archiving logs... 2024-04-02T15:43:19.897 DEBUG:teuthology.misc:Transferring archived files from smithi087:/var/log/ceph to /home/teuthworker/archive/gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi/7636172/remote/smithi087/log 2024-04-02T15:43:19.898 DEBUG:teuthology.orchestra.run.smithi087:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-02T15:43:20.426 DEBUG:teuthology.misc:Transferring archived files from smithi106:/var/log/ceph to /home/teuthworker/archive/gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi/7636172/remote/smithi106/log 2024-04-02T15:43:20.427 DEBUG:teuthology.orchestra.run.smithi106:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-02T15:43:20.809 INFO:tasks.cephadm:Removing cluster... 2024-04-02T15:43:20.809 DEBUG:teuthology.orchestra.run.smithi087:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 --force 2024-04-02T15:43:21.084 DEBUG:teuthology.orchestra.run.smithi106:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 --force 2024-04-02T15:43:21.364 INFO:tasks.cephadm:Removing cephadm ... 2024-04-02T15:43:21.364 DEBUG:teuthology.orchestra.run.smithi087:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-02T15:43:21.379 DEBUG:teuthology.orchestra.run.smithi106:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-02T15:43:21.394 INFO:tasks.cephadm:Teardown complete 2024-04-02T15:43:21.395 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-04-02T15:43:21.405 INFO:teuthology.task.clock:Checking final clock skew... 2024-04-02T15:43:21.405 DEBUG:teuthology.orchestra.run.smithi087:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-02T15:43:21.422 DEBUG:teuthology.orchestra.run.smithi106:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-02T15:43:21.433 INFO:teuthology.orchestra.run.smithi087.stderr:bash: line 1: ntpq: command not found 2024-04-02T15:43:21.437 INFO:teuthology.orchestra.run.smithi087.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-02T15:43:21.437 INFO:teuthology.orchestra.run.smithi087.stdout:=============================================================================== 2024-04-02T15:43:21.437 INFO:teuthology.orchestra.run.smithi087.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 61 -546us[ -546us] +/- 69ms 2024-04-02T15:43:21.437 INFO:teuthology.orchestra.run.smithi087.stdout:^* hv02.front.sepia.ceph.com 3 6 377 63 -254us[ -237us] +/- 74ms 2024-04-02T15:43:21.437 INFO:teuthology.orchestra.run.smithi087.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 62 +860us[ +860us] +/- 78ms 2024-04-02T15:43:21.437 INFO:teuthology.orchestra.run.smithi087.stdout:^? hv04.front.sepia.ceph.com 0 8 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T15:43:21.447 INFO:teuthology.orchestra.run.smithi106.stderr:bash: line 1: ntpq: command not found 2024-04-02T15:43:21.452 INFO:teuthology.orchestra.run.smithi106.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-02T15:43:21.452 INFO:teuthology.orchestra.run.smithi106.stdout:=============================================================================== 2024-04-02T15:43:21.452 INFO:teuthology.orchestra.run.smithi106.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 62 -535us[ -543us] +/- 69ms 2024-04-02T15:43:21.452 INFO:teuthology.orchestra.run.smithi106.stdout:^* hv02.front.sepia.ceph.com 3 6 377 62 -234us[ -241us] +/- 74ms 2024-04-02T15:43:21.452 INFO:teuthology.orchestra.run.smithi106.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 61 +872us[ +872us] +/- 78ms 2024-04-02T15:43:21.452 INFO:teuthology.orchestra.run.smithi106.stdout:^? hv04.front.sepia.ceph.com 0 8 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T15:43:21.453 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-02T15:43:21.462 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-02T15:43:21.463 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-02T15:43:21.471 DEBUG:teuthology.orchestra.run.smithi087:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-02T15:43:21.553 DEBUG:teuthology.orchestra.run.smithi106:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-02T15:43:21.623 DEBUG:teuthology.orchestra.run.smithi087:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-02T15:43:21.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-02T15:43:21.654 DEBUG:teuthology.orchestra.run.smithi106:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-02T15:43:21.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-02T15:43:21.683 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-02T15:43:21.694 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-02T15:43:21.702 INFO:teuthology.task.internal:Duration was 1421.549260 seconds 2024-04-02T15:43:21.703 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-02T15:43:21.710 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-02T15:43:21.710 DEBUG:teuthology.orchestra.run.smithi087:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-02T15:43:21.712 DEBUG:teuthology.orchestra.run.smithi106:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-02T15:43:21.754 INFO:teuthology.orchestra.run.smithi087.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-02T15:43:21.766 INFO:teuthology.orchestra.run.smithi106.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-02T15:43:22.163 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-02T15:43:22.163 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi087.front.sepia.ceph.com 2024-04-02T15:43:22.163 DEBUG:teuthology.orchestra.run.smithi087:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-02T15:43:22.182 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi106.front.sepia.ceph.com 2024-04-02T15:43:22.182 DEBUG:teuthology.orchestra.run.smithi106:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-02T15:43:22.222 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-02T15:43:22.223 DEBUG:teuthology.orchestra.run.smithi087:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-02T15:43:22.225 DEBUG:teuthology.orchestra.run.smithi106:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-02T15:43:22.292 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-02T15:43:22.292 DEBUG:teuthology.orchestra.run.smithi087:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-02T15:43:22.294 DEBUG:teuthology.orchestra.run.smithi106:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-02T15:43:22.710 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-02T15:43:22.736 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-02T15:43:22.736 DEBUG:teuthology.orchestra.run.smithi087:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-02T15:43:22.764 DEBUG:teuthology.orchestra.run.smithi106:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-02T15:43:22.793 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-02T15:43:22.808 DEBUG:teuthology.orchestra.run.smithi087:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-02T15:43:22.811 DEBUG:teuthology.orchestra.run.smithi106:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-02T15:43:22.834 INFO:teuthology.orchestra.run.smithi087.stdout:kernel.core_pattern = core 2024-04-02T15:43:22.860 INFO:teuthology.orchestra.run.smithi106.stdout:kernel.core_pattern = core 2024-04-02T15:43:22.914 DEBUG:teuthology.orchestra.run.smithi087:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-02T15:43:22.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-02T15:43:22.930 DEBUG:teuthology.orchestra.run.smithi106:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-02T15:43:22.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-02T15:43:22.971 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-02T15:43:22.984 INFO:teuthology.task.internal:Transferring archived files... 2024-04-02T15:43:22.985 DEBUG:teuthology.misc:Transferring archived files from smithi087:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi/7636172/remote/smithi087 2024-04-02T15:43:22.985 DEBUG:teuthology.orchestra.run.smithi087:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-02T15:43:23.035 DEBUG:teuthology.misc:Transferring archived files from smithi106:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi/7636172/remote/smithi106 2024-04-02T15:43:23.035 DEBUG:teuthology.orchestra.run.smithi106:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-02T15:43:23.081 INFO:teuthology.task.internal:Removing archive directory... 2024-04-02T15:43:23.081 DEBUG:teuthology.orchestra.run.smithi087:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-02T15:43:23.084 DEBUG:teuthology.orchestra.run.smithi106:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-02T15:43:23.126 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-02T15:43:23.156 INFO:teuthology.task.internal:Not uploading archives. 2024-04-02T15:43:23.156 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-02T15:43:23.191 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-02T15:43:23.191 DEBUG:teuthology.orchestra.run.smithi087:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-02T15:43:23.194 DEBUG:teuthology.orchestra.run.smithi106:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-02T15:43:23.206 INFO:teuthology.orchestra.run.smithi087.stdout: 265305 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 2 15:43 /home/ubuntu/cephtest 2024-04-02T15:43:23.208 INFO:teuthology.orchestra.run.smithi106.stdout: 265955 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 2 15:43 /home/ubuntu/cephtest 2024-04-02T15:43:23.209 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-02T15:43:23.229 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-02T15:43:23.392 INFO:teuthology.nuke:Checking targets against current locks 2024-04-02T15:43:23.418 DEBUG:teuthology.nuke:shortname: smithi087 2024-04-02T15:43:23.419 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-02T15:43:23.440 DEBUG:teuthology.nuke:shortname: smithi106 2024-04-02T15:43:23.440 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-02T15:43:23.444 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi087.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi/7636172', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-02 15:11:28.953202', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGxSrXeQXaQfffNLByQxohUKCY/iDJzmwOw0ad1+nmK0inhxPAWMyoRY28TeUDt7GkYdTi34J5VcVGYzc+/gPxc='} 2024-04-02T15:43:23.462 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi106.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-04-02_14:30:22-orch:cephadm-wip-guits-testing-2024-04-02-0703-distro-default-smithi/7636172', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-02 15:11:28.954825', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPa6mpToccO2yFYP2iF9WIzT2AlXu5XNyUVL5qlsL30EiS/EKea53h9/0CSO34PPwfLbbXDhVgYjx62pVH+V5ag='} 2024-04-02T15:43:23.489 INFO:teuthology.orchestra.console.smithi087:Power off 2024-04-02T15:43:23.490 DEBUG:teuthology.orchestra.console.smithi087:pexpect command: ipmitool -H smithi087.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-02T15:43:23.515 DEBUG:teuthology.orchestra.console.smithi087:power off output: Chassis Power Control: Down/Off 2024-04-02T15:43:23.516 DEBUG:teuthology.orchestra.console.smithi087:pexpect command: ipmitool -H smithi087.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-02T15:43:23.532 INFO:teuthology.orchestra.console.smithi106:Power off 2024-04-02T15:43:23.532 DEBUG:teuthology.orchestra.console.smithi106:pexpect command: ipmitool -H smithi106.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-02T15:43:23.541 DEBUG:teuthology.orchestra.console.smithi087:check power output: Chassis Power is on 2024-04-02T15:43:23.556 DEBUG:teuthology.orchestra.console.smithi106:power off output: Chassis Power Control: Down/Off 2024-04-02T15:43:23.556 DEBUG:teuthology.orchestra.console.smithi106:pexpect command: ipmitool -H smithi106.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-02T15:43:23.579 DEBUG:teuthology.orchestra.console.smithi106:check power output: Chassis Power is on 2024-04-02T15:43:27.542 DEBUG:teuthology.orchestra.console.smithi087:pexpect command: ipmitool -H smithi087.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-02T15:43:27.580 DEBUG:teuthology.orchestra.console.smithi106:pexpect command: ipmitool -H smithi106.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-02T15:43:27.653 DEBUG:teuthology.orchestra.console.smithi087:check power output: Chassis Power is on 2024-04-02T15:43:27.690 DEBUG:teuthology.orchestra.console.smithi106:check power output: Chassis Power is on 2024-04-02T15:43:31.658 DEBUG:teuthology.orchestra.console.smithi087:pexpect command: ipmitool -H smithi087.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-02T15:43:31.691 DEBUG:teuthology.orchestra.console.smithi106:pexpect command: ipmitool -H smithi106.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-02T15:43:31.769 DEBUG:teuthology.orchestra.console.smithi087:check power output: Chassis Power is off 2024-04-02T15:43:31.801 DEBUG:teuthology.orchestra.console.smithi106:check power output: Chassis Power is on 2024-04-02T15:43:31.870 INFO:teuthology.orchestra.console.smithi087:Power off completed 2024-04-02T15:43:35.805 DEBUG:teuthology.orchestra.console.smithi106:pexpect command: ipmitool -H smithi106.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-02T15:43:35.915 DEBUG:teuthology.orchestra.console.smithi106:check power output: Chassis Power is off 2024-04-02T15:43:36.016 INFO:teuthology.orchestra.console.smithi106:Power off completed 2024-04-02T15:43:36.121 INFO:teuthology.run:Summary data: description: orch:cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/on mon_election/classic} duration: 1421.5492599010468 failure_reason: 'Command failed on smithi087 with status 22: ''sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 0400842c-f105-11ee-b647-cb9ed24678a4 -e sha1=3f3f3ed992429b85be729743ad3b39d52dd90744 -- bash -c \''ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1\''''' owner: scheduled_gabrioux@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=d8f30325fb784214b57dd646d1c9fe8f status: fail success: false 2024-04-02T15:43:36.121 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-02T15:43:36.199 INFO:teuthology.run:FAIL