2024-04-07T14:22:35.516 INFO:root:teuthology version: 0.0.1.dev262+ge691533 2024-04-07T14:22:35.516 DEBUG:teuthology.run:Teuthology command: teuthology --name teuthology-2024-04-01_20:08:13-orch-main-distro-default-smithi --owner scheduled_teuthology@teuthology --archive /home/teuthworker/archive/teuthology-2024-04-01_20:08:13-orch-main-distro-default-smithi/7634198 --verbose --description orch/cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/repo_digest 3-upgrade/staggered 4-wait 5-upgrade-ls agent/on mon_election/connectivity} -- /home/teuthworker/archive/teuthology-2024-04-01_20:08:13-orch-main-distro-default-smithi/7634198/orig.config.yaml 2024-04-07T14:22:35.559 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-07T14:22:35.644 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-04-01_20:08:13-orch-main-distro-default-smithi/7634198 branch: main description: orch/cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/repo_digest 3-upgrade/staggered 4-wait 5-upgrade-ls agent/on mon_election/connectivity} email: ceph-qa@ceph.com first_in_suite: false job_id: '7634198' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi name: teuthology-2024-04-01_20:08:13-orch-main-distro-default-smithi no_nested_subset: false nuke-on-error: true os_type: centos os_version: 9.stream overrides: admin_socket: branch: main ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mgr/cephadm/use_agent: true mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: e5c885fe95ee982ff7b45214daa61b1a6d09d57d ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: e5c885fe95ee982ff7b45214daa61b1a6d09d57d workunit: branch: main sha1: e5c885fe95ee982ff7b45214daa61b1a6d09d57d owner: scheduled_teuthology@teuthology priority: 950 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b seed: 7151 sha1: e5c885fe95ee982ff7b45214daa61b1a6d09d57d sleep_before_teardown: 0 subset: 39/64 suite: orch suite_branch: main suite_path: /home/teuthworker/src/git.ceph.com_ceph_e5c885fe95ee982ff7b45214daa61b1a6d09d57d/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: e5c885fe95ee982ff7b45214daa61b1a6d09d57d targets: smithi083.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAfjhHyLOYAGOtB1YzxEgzc07ihb/9sedUE5obRqGNqEj3nlEEgCNd9gfUZYMGlvZzmGXYoun+d6e7CX6G6X4mk= smithi189.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKjMxsrnDBjL2m/RVeBcTmi2M6kn6U6nmOsSBocU6Y+64PjMMqutRuS9mxF5xWLknvGcAlMMyjpNkUrfVBhuFH0= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 - cephadm.shell: mon.a: - ceph config set mgr mgr/cephadm/use_repo_digest false --force - cephadm.shell: env: - sha1 mon.a: - radosgw-admin realm create --rgw-realm=r --default - radosgw-admin zonegroup create --rgw-zonegroup=default --master --default - radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default - radosgw-admin period update --rgw-realm=r --commit - ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000 - ceph osd pool create foo - rbd pool init foo - ceph orch apply iscsi foo u p - sleep 180 - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph orch ps - ceph versions - ceph -s - ceph orch ls - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 2' - ceph mgr fail - sleep 180 - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph health detail - ceph -s - ceph mgr fail - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 1' - ceph mgr fail - sleep 180 - ceph orch ps - ceph orch ls - ceph versions - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph versions | jq -e '.mgr | length == 1' - ceph versions | jq -e '.mgr | keys' | grep $sha1 - ceph versions | jq -e '.overall | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 1' - ceph versions | jq -e '.mon | keys' | grep $sha1 - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 5' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 7' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 8' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 1' - ceph versions | jq -e '.osd | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.rgw | length == 1' - ceph versions | jq -e '.rgw | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 mon.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph versions - ceph orch upgrade status - ceph health detail - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - ceph orch ls | grep '^osd ' - cephadm.shell: mon.a: - ceph orch upgrade ls - ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0 - ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2 teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: e691533f9cbb33d85b2187bba20d7102f098636d timestamp: 2024-04-01_20:08:13 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-04-07T14:22:35.644 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_e5c885fe95ee982ff7b45214daa61b1a6d09d57d/qa; will attempt to use it 2024-04-07T14:22:35.645 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_e5c885fe95ee982ff7b45214daa61b1a6d09d57d/qa/tasks 2024-04-07T14:22:35.645 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-07T14:22:35.646 INFO:teuthology.task.internal:Checking packages... 2024-04-07T14:22:35.673 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'e5c885fe95ee982ff7b45214daa61b1a6d09d57d' 2024-04-07T14:22:35.674 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-07T14:22:35.674 INFO:teuthology.packaging:ref: None 2024-04-07T14:22:35.674 INFO:teuthology.packaging:tag: None 2024-04-07T14:22:35.674 INFO:teuthology.packaging:branch: main 2024-04-07T14:22:35.674 INFO:teuthology.packaging:sha1: e5c885fe95ee982ff7b45214daa61b1a6d09d57d 2024-04-07T14:22:35.674 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=main 2024-04-07T14:22:35.922 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-2778.ga971fd5f 2024-04-07T14:22:35.923 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-07T14:22:35.930 INFO:teuthology.task.internal:no buildpackages task found 2024-04-07T14:22:35.931 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-07T14:22:35.937 INFO:teuthology.task.internal:Saving configuration 2024-04-07T14:22:35.955 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-07T14:22:35.961 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-07T14:22:35.988 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi083.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-01_20:08:13-orch-main-distro-default-smithi/7634198', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-07 14:17:18.353314', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAfjhHyLOYAGOtB1YzxEgzc07ihb/9sedUE5obRqGNqEj3nlEEgCNd9gfUZYMGlvZzmGXYoun+d6e7CX6G6X4mk='} 2024-04-07T14:22:36.013 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi189.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-01_20:08:13-orch-main-distro-default-smithi/7634198', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-07 14:17:18.351773', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKjMxsrnDBjL2m/RVeBcTmi2M6kn6U6nmOsSBocU6Y+64PjMMqutRuS9mxF5xWLknvGcAlMMyjpNkUrfVBhuFH0='} 2024-04-07T14:22:36.013 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-07T14:22:36.019 INFO:teuthology.task.internal:roles: ubuntu@smithi083.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-04-07T14:22:36.019 INFO:teuthology.task.internal:roles: ubuntu@smithi189.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-04-07T14:22:36.019 INFO:teuthology.run_tasks:Running task console_log... 2024-04-07T14:22:36.094 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f4992eac0d0>, signals=[15]) 2024-04-07T14:22:36.094 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-07T14:22:36.100 INFO:teuthology.task.internal:Opening connections... 2024-04-07T14:22:36.100 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi083.front.sepia.ceph.com 2024-04-07T14:22:36.102 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi083.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-07T14:22:36.193 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi189.front.sepia.ceph.com 2024-04-07T14:22:36.194 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi189.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-07T14:22:36.283 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-07T14:22:36.290 DEBUG:teuthology.orchestra.run.smithi083:> uname -m 2024-04-07T14:22:36.307 INFO:teuthology.orchestra.run.smithi083.stdout:x86_64 2024-04-07T14:22:36.308 DEBUG:teuthology.orchestra.run.smithi083:> cat /etc/os-release 2024-04-07T14:22:36.364 INFO:teuthology.orchestra.run.smithi083.stdout:NAME="CentOS Stream" 2024-04-07T14:22:36.364 INFO:teuthology.orchestra.run.smithi083.stdout:VERSION="9" 2024-04-07T14:22:36.364 INFO:teuthology.orchestra.run.smithi083.stdout:ID="centos" 2024-04-07T14:22:36.364 INFO:teuthology.orchestra.run.smithi083.stdout:ID_LIKE="rhel fedora" 2024-04-07T14:22:36.364 INFO:teuthology.orchestra.run.smithi083.stdout:VERSION_ID="9" 2024-04-07T14:22:36.364 INFO:teuthology.orchestra.run.smithi083.stdout:PLATFORM_ID="platform:el9" 2024-04-07T14:22:36.364 INFO:teuthology.orchestra.run.smithi083.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-07T14:22:36.364 INFO:teuthology.orchestra.run.smithi083.stdout:ANSI_COLOR="0;31" 2024-04-07T14:22:36.364 INFO:teuthology.orchestra.run.smithi083.stdout:LOGO="fedora-logo-icon" 2024-04-07T14:22:36.365 INFO:teuthology.orchestra.run.smithi083.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-07T14:22:36.365 INFO:teuthology.orchestra.run.smithi083.stdout:HOME_URL="https://centos.org/" 2024-04-07T14:22:36.365 INFO:teuthology.orchestra.run.smithi083.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-07T14:22:36.365 INFO:teuthology.orchestra.run.smithi083.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-07T14:22:36.365 INFO:teuthology.orchestra.run.smithi083.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-07T14:22:36.365 INFO:teuthology.lock.ops:Updating smithi083.front.sepia.ceph.com on lock server 2024-04-07T14:22:36.383 DEBUG:teuthology.orchestra.run.smithi189:> uname -m 2024-04-07T14:22:36.401 INFO:teuthology.orchestra.run.smithi189.stdout:x86_64 2024-04-07T14:22:36.401 DEBUG:teuthology.orchestra.run.smithi189:> cat /etc/os-release 2024-04-07T14:22:36.458 INFO:teuthology.orchestra.run.smithi189.stdout:NAME="CentOS Stream" 2024-04-07T14:22:36.458 INFO:teuthology.orchestra.run.smithi189.stdout:VERSION="9" 2024-04-07T14:22:36.458 INFO:teuthology.orchestra.run.smithi189.stdout:ID="centos" 2024-04-07T14:22:36.458 INFO:teuthology.orchestra.run.smithi189.stdout:ID_LIKE="rhel fedora" 2024-04-07T14:22:36.458 INFO:teuthology.orchestra.run.smithi189.stdout:VERSION_ID="9" 2024-04-07T14:22:36.458 INFO:teuthology.orchestra.run.smithi189.stdout:PLATFORM_ID="platform:el9" 2024-04-07T14:22:36.459 INFO:teuthology.orchestra.run.smithi189.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-07T14:22:36.459 INFO:teuthology.orchestra.run.smithi189.stdout:ANSI_COLOR="0;31" 2024-04-07T14:22:36.459 INFO:teuthology.orchestra.run.smithi189.stdout:LOGO="fedora-logo-icon" 2024-04-07T14:22:36.459 INFO:teuthology.orchestra.run.smithi189.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-07T14:22:36.459 INFO:teuthology.orchestra.run.smithi189.stdout:HOME_URL="https://centos.org/" 2024-04-07T14:22:36.459 INFO:teuthology.orchestra.run.smithi189.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-07T14:22:36.459 INFO:teuthology.orchestra.run.smithi189.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-07T14:22:36.459 INFO:teuthology.orchestra.run.smithi189.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-07T14:22:36.459 INFO:teuthology.lock.ops:Updating smithi189.front.sepia.ceph.com on lock server 2024-04-07T14:22:36.483 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-07T14:22:36.492 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-07T14:22:36.510 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-07T14:22:36.510 DEBUG:teuthology.orchestra.run.smithi083:> test '!' -e /home/ubuntu/cephtest 2024-04-07T14:22:36.513 DEBUG:teuthology.orchestra.run.smithi189:> test '!' -e /home/ubuntu/cephtest 2024-04-07T14:22:36.529 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-07T14:22:36.536 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-07T14:22:36.536 DEBUG:teuthology.orchestra.run.smithi083:> test -z $(ls -A /var/lib/ceph) 2024-04-07T14:22:36.570 DEBUG:teuthology.orchestra.run.smithi189:> test -z $(ls -A /var/lib/ceph) 2024-04-07T14:22:36.660 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-07T14:22:36.713 INFO:teuthology.run_tasks:Running task kernel... 2024-04-07T14:22:36.727 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2024-04-07T14:22:36.727 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2024-04-07T14:22:36.728 DEBUG:teuthology.orchestra.run.smithi083:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-07T14:22:36.728 DEBUG:teuthology.orchestra.run.smithi189:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-07T14:22:36.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-07T14:22:36.744 DEBUG:teuthology.orchestra.run.smithi083:> uname -r 2024-04-07T14:22:36.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-07T14:22:36.745 DEBUG:teuthology.orchestra.run.smithi189:> uname -r 2024-04-07T14:22:36.801 INFO:teuthology.orchestra.run.smithi083.stdout:5.14.0-361.el9.x86_64 2024-04-07T14:22:36.801 INFO:teuthology.task.kernel:Running kernel on smithi083: 5.14.0-361.el9.x86_64 2024-04-07T14:22:36.802 DEBUG:teuthology.orchestra.run.smithi083:> sudo yum install -y kernel 2024-04-07T14:22:36.803 INFO:teuthology.orchestra.run.smithi189.stdout:5.14.0-361.el9.x86_64 2024-04-07T14:22:36.803 INFO:teuthology.task.kernel:Running kernel on smithi189: 5.14.0-361.el9.x86_64 2024-04-07T14:22:36.803 DEBUG:teuthology.orchestra.run.smithi189:> sudo yum install -y kernel 2024-04-07T14:22:38.923 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS-9 - AppStream 17 MB/s | 8.1 MB 00:00 2024-04-07T14:22:39.106 INFO:teuthology.orchestra.run.smithi189.stdout:CentOS-9 - AppStream 14 MB/s | 8.1 MB 00:00 2024-04-07T14:22:40.724 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS-9 - BaseOS 5.1 MB/s | 2.5 MB 00:00 2024-04-07T14:22:40.940 INFO:teuthology.orchestra.run.smithi189.stdout:CentOS-9 - BaseOS 4.9 MB/s | 2.5 MB 00:00 2024-04-07T14:22:41.814 INFO:teuthology.orchestra.run.smithi189.stdout:CentOS Stream 9 - BaseOS 19 MB/s | 8.0 MB 00:00 2024-04-07T14:22:41.858 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS Stream 9 - BaseOS 12 MB/s | 8.0 MB 00:00 2024-04-07T14:22:43.951 INFO:teuthology.orchestra.run.smithi189.stdout:CentOS Stream 9 - AppStream 18 MB/s | 19 MB 00:01 2024-04-07T14:22:48.151 INFO:teuthology.orchestra.run.smithi189.stdout:CentOS Stream 9 - CRB 8.8 MB/s | 6.1 MB 00:00 2024-04-07T14:22:49.659 INFO:teuthology.orchestra.run.smithi189.stdout:CentOS Stream 9 - Extras packages 44 kB/s | 16 kB 00:00 2024-04-07T14:22:50.223 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS Stream 9 - AppStream 2.6 MB/s | 19 MB 00:07 2024-04-07T14:22:51.920 INFO:teuthology.orchestra.run.smithi189.stdout:Extra Packages for Enterprise Linux 9.8 MB/s | 21 MB 00:02 2024-04-07T14:22:54.974 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS Stream 9 - CRB 5.2 MB/s | 6.1 MB 00:01 2024-04-07T14:22:56.475 INFO:teuthology.orchestra.run.smithi083.stdout:CentOS Stream 9 - Extras packages 45 kB/s | 16 kB 00:00 2024-04-07T14:22:56.918 INFO:teuthology.orchestra.run.smithi189.stdout:lab-extras 40 kB/s | 1.7 kB 00:00 2024-04-07T14:22:58.828 INFO:teuthology.orchestra.run.smithi083.stdout:Extra Packages for Enterprise Linux 9.4 MB/s | 21 MB 00:02 2024-04-07T14:22:59.059 INFO:teuthology.orchestra.run.smithi189.stderr:Modular dependency problems: 2024-04-07T14:22:59.060 INFO:teuthology.orchestra.run.smithi189.stderr: 2024-04-07T14:22:59.060 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 1: conflicting requests 2024-04-07T14:22:59.060 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.060 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 2: conflicting requests 2024-04-07T14:22:59.060 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.060 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 3: conflicting requests 2024-04-07T14:22:59.060 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.060 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 4: conflicting requests 2024-04-07T14:22:59.060 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.060 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 5: conflicting requests 2024-04-07T14:22:59.060 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.060 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 6: conflicting requests 2024-04-07T14:22:59.061 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.061 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 7: conflicting requests 2024-04-07T14:22:59.061 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.061 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 8: conflicting requests 2024-04-07T14:22:59.061 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.061 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 9: conflicting requests 2024-04-07T14:22:59.061 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.061 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 10: conflicting requests 2024-04-07T14:22:59.061 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.061 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 11: conflicting requests 2024-04-07T14:22:59.061 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.061 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 12: conflicting requests 2024-04-07T14:22:59.061 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.061 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 13: conflicting requests 2024-04-07T14:22:59.061 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.062 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 14: conflicting requests 2024-04-07T14:22:59.062 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.062 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 15: conflicting requests 2024-04-07T14:22:59.062 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.062 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 16: conflicting requests 2024-04-07T14:22:59.062 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.062 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 17: conflicting requests 2024-04-07T14:22:59.062 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.062 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 18: conflicting requests 2024-04-07T14:22:59.062 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.062 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 19: conflicting requests 2024-04-07T14:22:59.062 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.062 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 20: conflicting requests 2024-04-07T14:22:59.063 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.063 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 21: conflicting requests 2024-04-07T14:22:59.063 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.063 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 22: conflicting requests 2024-04-07T14:22:59.063 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.063 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 23: conflicting requests 2024-04-07T14:22:59.063 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.063 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 24: conflicting requests 2024-04-07T14:22:59.063 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.063 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 25: conflicting requests 2024-04-07T14:22:59.063 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.063 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 26: conflicting requests 2024-04-07T14:22:59.063 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.063 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 27: conflicting requests 2024-04-07T14:22:59.064 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.064 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 28: conflicting requests 2024-04-07T14:22:59.064 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.064 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 29: conflicting requests 2024-04-07T14:22:59.064 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.064 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 30: conflicting requests 2024-04-07T14:22:59.064 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.064 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 31: conflicting requests 2024-04-07T14:22:59.064 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.064 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 32: conflicting requests 2024-04-07T14:22:59.064 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.064 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 33: conflicting requests 2024-04-07T14:22:59.064 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.064 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 34: conflicting requests 2024-04-07T14:22:59.064 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.064 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 35: conflicting requests 2024-04-07T14:22:59.065 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.065 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 36: conflicting requests 2024-04-07T14:22:59.065 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.065 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 37: conflicting requests 2024-04-07T14:22:59.065 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.065 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.065 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.065 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 38: cannot install the best candidate for the job 2024-04-07T14:22:59.065 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.065 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.065 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.065 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.065 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 39: conflicting requests 2024-04-07T14:22:59.065 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.065 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.066 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.066 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.066 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.066 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.066 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 40: cannot install the best candidate for the job 2024-04-07T14:22:59.066 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.066 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.066 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.066 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.066 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 41: cannot install the best candidate for the job 2024-04-07T14:22:59.066 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.066 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.066 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.066 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.067 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 42: cannot install the best candidate for the job 2024-04-07T14:22:59.067 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.067 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.067 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.067 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.067 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 43: cannot install the best candidate for the job 2024-04-07T14:22:59.067 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.067 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.067 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.067 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.067 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 44: cannot install the best candidate for the job 2024-04-07T14:22:59.067 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.067 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.067 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.067 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.068 INFO:teuthology.orchestra.run.smithi189.stderr: Problem 45: cannot install the best candidate for the job 2024-04-07T14:22:59.068 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.068 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.068 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.068 INFO:teuthology.orchestra.run.smithi189.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-07T14:22:59.425 INFO:teuthology.orchestra.run.smithi189.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-07T14:22:59.425 INFO:teuthology.orchestra.run.smithi189.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-04-07T14:22:59.521 INFO:teuthology.orchestra.run.smithi189.stdout:Dependencies resolved. 2024-04-07T14:22:59.530 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-04-07T14:22:59.530 INFO:teuthology.orchestra.run.smithi189.stdout: Package Architecture Version Repository Size 2024-04-07T14:22:59.530 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-04-07T14:22:59.530 INFO:teuthology.orchestra.run.smithi189.stdout:Installing: 2024-04-07T14:22:59.531 INFO:teuthology.orchestra.run.smithi189.stdout: kernel x86_64 5.14.0-432.el9 baseos 66 k 2024-04-07T14:22:59.531 INFO:teuthology.orchestra.run.smithi189.stdout:Installing dependencies: 2024-04-07T14:22:59.531 INFO:teuthology.orchestra.run.smithi189.stdout: kernel-core x86_64 5.14.0-432.el9 baseos 15 M 2024-04-07T14:22:59.531 INFO:teuthology.orchestra.run.smithi189.stdout: kernel-modules x86_64 5.14.0-432.el9 baseos 33 M 2024-04-07T14:22:59.531 INFO:teuthology.orchestra.run.smithi189.stdout: kernel-modules-core x86_64 5.14.0-432.el9 baseos 27 M 2024-04-07T14:22:59.531 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-04-07T14:22:59.531 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction Summary 2024-04-07T14:22:59.531 INFO:teuthology.orchestra.run.smithi189.stdout:================================================================================ 2024-04-07T14:22:59.531 INFO:teuthology.orchestra.run.smithi189.stdout:Install 4 Packages 2024-04-07T14:22:59.531 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-04-07T14:22:59.532 INFO:teuthology.orchestra.run.smithi189.stdout:Total download size: 76 M 2024-04-07T14:22:59.532 INFO:teuthology.orchestra.run.smithi189.stdout:Installed size: 123 M 2024-04-07T14:22:59.532 INFO:teuthology.orchestra.run.smithi189.stdout:Downloading Packages: 2024-04-07T14:23:00.035 INFO:teuthology.orchestra.run.smithi189.stdout:(1/4): kernel-5.14.0-432.el9.x86_64.rpm 377 kB/s | 66 kB 00:00 2024-04-07T14:23:00.623 INFO:teuthology.orchestra.run.smithi189.stdout:(2/4): kernel-core-5.14.0-432.el9.x86_64.rpm 20 MB/s | 15 MB 00:00 2024-04-07T14:23:01.403 INFO:teuthology.orchestra.run.smithi189.stdout:(3/4): kernel-modules-core-5.14.0-432.el9.x86_6 20 MB/s | 27 MB 00:01 2024-04-07T14:23:02.211 INFO:teuthology.orchestra.run.smithi189.stdout:(4/4): kernel-modules-5.14.0-432.el9.x86_64.rpm 14 MB/s | 33 MB 00:02 2024-04-07T14:23:02.211 INFO:teuthology.orchestra.run.smithi189.stdout:-------------------------------------------------------------------------------- 2024-04-07T14:23:02.212 INFO:teuthology.orchestra.run.smithi189.stdout:Total 28 MB/s | 76 MB 00:02 2024-04-07T14:23:02.875 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction check 2024-04-07T14:23:03.238 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction check succeeded. 2024-04-07T14:23:03.239 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction test 2024-04-07T14:23:03.867 INFO:teuthology.orchestra.run.smithi083.stdout:lab-extras 12 kB/s | 1.7 kB 00:00 2024-04-07T14:23:04.251 INFO:teuthology.orchestra.run.smithi189.stdout:Transaction test succeeded. 2024-04-07T14:23:04.252 INFO:teuthology.orchestra.run.smithi189.stdout:Running transaction 2024-04-07T14:23:05.993 INFO:teuthology.orchestra.run.smithi083.stderr:Modular dependency problems: 2024-04-07T14:23:05.994 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-04-07T14:23:05.994 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 1: conflicting requests 2024-04-07T14:23:05.994 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module container-tools:rhel8:820240321091303:20125149.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.994 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 2: conflicting requests 2024-04-07T14:23:05.994 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module eclipse:rhel8:820201023100746:b230ed4e.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.994 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 3: conflicting requests 2024-04-07T14:23:05.994 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module freeradius:3.0:8080020230117180605:89170a74.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.994 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 4: conflicting requests 2024-04-07T14:23:05.994 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module gimp:2.8:8000020190628145146:4148dfdf.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.994 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 5: conflicting requests 2024-04-07T14:23:05.994 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module go-toolset:rhel8:820240401160413:b754926a.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.994 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 6: conflicting requests 2024-04-07T14:23:05.995 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module httpd:2.4:820240216084734:9edba152.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.995 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 7: conflicting requests 2024-04-07T14:23:05.995 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module idm:client:820240307184541:49cc9d1b.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.995 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 8: conflicting requests 2024-04-07T14:23:05.995 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module inkscape:0.92.3:8080020221205124429:3e031279.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.995 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 9: conflicting requests 2024-04-07T14:23:05.995 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module jmc:rhel8:8050020211110222101:6392b1f8.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.995 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 10: conflicting requests 2024-04-07T14:23:05.995 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module llvm-toolset:rhel8:820240205133135:9edba152.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.995 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 11: conflicting requests 2024-04-07T14:23:05.995 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module log4j:2:8080020221205124743:9d367344.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.995 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 12: conflicting requests 2024-04-07T14:23:05.995 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module mailman:2.1:820230727085757:77fc8825.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.995 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 13: conflicting requests 2024-04-07T14:23:05.996 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module mariadb:10.3:8030020210419150013:30b713e6.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.996 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 14: conflicting requests 2024-04-07T14:23:05.996 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module maven:3.5:8060020211203142015:c0229ad2.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.996 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 15: conflicting requests 2024-04-07T14:23:05.996 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module mercurial:4.8:8000020190628020724:4148dfdf.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.996 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 16: conflicting requests 2024-04-07T14:23:05.996 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module mod_auth_openidc:2.3:820230721123832:9edba152.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.996 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 17: conflicting requests 2024-04-07T14:23:05.996 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module mysql:8.0:8040020210901180257:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.996 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 18: conflicting requests 2024-04-07T14:23:05.996 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module nginx:1.14:8000020211221191913:55190bc5.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.996 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 19: conflicting requests 2024-04-07T14:23:05.996 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module nodejs:10:8040020210120182536:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.996 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 20: conflicting requests 2024-04-07T14:23:05.996 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module perl:5.26:8000020190628020724:55190bc5.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.997 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 21: conflicting requests 2024-04-07T14:23:05.997 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module php:7.2:8020020200507003613:2c7ca891.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.997 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 22: conflicting requests 2024-04-07T14:23:05.997 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module postgresql:10:820230223114800:9edba152.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.997 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 23: conflicting requests 2024-04-07T14:23:05.997 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module python27:2.7:820240208011952:182f7c73.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.997 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 24: conflicting requests 2024-04-07T14:23:05.997 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module python36:3.6:8050020210825152031:982725ab.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.997 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 25: conflicting requests 2024-04-07T14:23:05.997 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module python38:3.8:820230810143931:64c1cd5a.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.997 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 26: conflicting requests 2024-04-07T14:23:05.997 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module python39:3.9:820240214182535:17377f89.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.997 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 27: conflicting requests 2024-04-07T14:23:05.997 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module redis:5:8040020211019153849:9f9e2e7e.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.997 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 28: conflicting requests 2024-04-07T14:23:05.997 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module rhn-tools:1.0:8010020191114034948:f69d1239.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.998 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 29: conflicting requests 2024-04-07T14:23:05.998 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module ruby:2.5:820230627084142:9edba152.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.998 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 30: conflicting requests 2024-04-07T14:23:05.998 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module rust-toolset:rhel8:820240119204620:b09eea91.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.998 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 31: conflicting requests 2024-04-07T14:23:05.998 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module satellite-5-client:1.0:8010020191114035551:cdc1202b.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.998 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 32: conflicting requests 2024-04-07T14:23:05.998 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module squid:4:820240319173245:9edba152.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.998 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 33: conflicting requests 2024-04-07T14:23:05.998 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module subversion:1.10:8070020220711155714:78111232.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.998 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 34: conflicting requests 2024-04-07T14:23:05.998 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module swig:3.0:8030020200706142531:30b713e6.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.998 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 35: conflicting requests 2024-04-07T14:23:05.998 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module varnish:6:8050020211109225449:b4937e53.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.998 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 36: conflicting requests 2024-04-07T14:23:05.999 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module virt:rhel:820240314161907:9edba152.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.999 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 37: conflicting requests 2024-04-07T14:23:05.999 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:7cc0a66d.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.999 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:8030020200716155257:b967a9a2.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.999 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module perl-libwww-perl:6.34:820230811133638:36f9fae6.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.999 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 38: cannot install the best candidate for the job 2024-04-07T14:23:05.999 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:a5949e2e.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.999 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:8010020191114031501:be2e4737.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.999 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:2ab761e1.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.999 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module perl-YAML:1.24:820230223114759:86e45846.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.999 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 39: conflicting requests 2024-04-07T14:23:05.999 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.999 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:8030020200715230104:ea09926d.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.999 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:082fdf2f.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.999 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-07T14:23:05.999 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-07T14:23:06.000 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module perl-IO-Socket-SSL:2.066:820230223114759:fbe42456.x86_64 from CentOS-AppStream 2024-04-07T14:23:06.000 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 40: cannot install the best candidate for the job 2024-04-07T14:23:06.000 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:16b3ab4d.x86_64 from CentOS-AppStream 2024-04-07T14:23:06.000 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:8010020191114031513:e1f37755.x86_64 from CentOS-AppStream 2024-04-07T14:23:06.000 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-07T14:23:06.000 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module perl-FCGI:0.78:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-07T14:23:06.000 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 41: cannot install the best candidate for the job 2024-04-07T14:23:06.000 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:1e4bbb35.x86_64 from CentOS-AppStream 2024-04-07T14:23:06.000 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:8030020200716150652:ea09926d.x86_64 from CentOS-AppStream 2024-04-07T14:23:06.000 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:0ccef39c.x86_64 from CentOS-AppStream 2024-04-07T14:23:06.000 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module perl-DBI:1.641:820230223114759:dbc5bc9a.x86_64 from CentOS-AppStream 2024-04-07T14:23:06.000 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 42: cannot install the best candidate for the job 2024-04-07T14:23:06.000 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:3a70019f.x86_64 from CentOS-AppStream 2024-04-07T14:23:06.000 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:8030020200716174729:cccafca5.x86_64 from CentOS-AppStream 2024-04-07T14:23:06.000 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-07T14:23:06.001 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-SQLite:1.58:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-07T14:23:06.001 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 43: cannot install the best candidate for the job 2024-04-07T14:23:06.001 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:56fce90f.x86_64 from CentOS-AppStream 2024-04-07T14:23:06.001 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:8030020200716213634:98a3c9d0.x86_64 from CentOS-AppStream 2024-04-07T14:23:06.001 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:06b7596e.x86_64 from CentOS-AppStream 2024-04-07T14:23:06.001 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-Pg:3.7:820230223114759:d2fc5d76.x86_64 from CentOS-AppStream 2024-04-07T14:23:06.001 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 44: cannot install the best candidate for the job 2024-04-07T14:23:06.001 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:3a70019f.x86_64 from CentOS-AppStream 2024-04-07T14:23:06.001 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:8030020200716171139:cccafca5.x86_64 from CentOS-AppStream 2024-04-07T14:23:06.001 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:6fdf7e5d.x86_64 from CentOS-AppStream 2024-04-07T14:23:06.001 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module perl-DBD-MySQL:4.046:820230223114759:d4326aba.x86_64 from CentOS-AppStream 2024-04-07T14:23:06.001 INFO:teuthology.orchestra.run.smithi083.stderr: Problem 45: cannot install the best candidate for the job 2024-04-07T14:23:06.001 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8010020191120175858:95fce66d.x86_64 from CentOS-AppStream 2024-04-07T14:23:06.001 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:8030020200723233049:09acf126.x86_64 from CentOS-AppStream 2024-04-07T14:23:06.001 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:436af1ae.x86_64 from CentOS-AppStream 2024-04-07T14:23:06.001 INFO:teuthology.orchestra.run.smithi083.stderr: - nothing provides module(platform:el8) needed by module perl-App-cpanminus:1.7044:820230223114759:b12195ea.x86_64 from CentOS-AppStream 2024-04-07T14:23:06.416 INFO:teuthology.orchestra.run.smithi083.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-07T14:23:06.416 INFO:teuthology.orchestra.run.smithi083.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-04-07T14:23:06.531 INFO:teuthology.orchestra.run.smithi083.stdout:Dependencies resolved. 2024-04-07T14:23:06.541 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-04-07T14:23:06.541 INFO:teuthology.orchestra.run.smithi083.stdout: Package Architecture Version Repository Size 2024-04-07T14:23:06.541 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-04-07T14:23:06.541 INFO:teuthology.orchestra.run.smithi083.stdout:Installing: 2024-04-07T14:23:06.541 INFO:teuthology.orchestra.run.smithi083.stdout: kernel x86_64 5.14.0-432.el9 baseos 66 k 2024-04-07T14:23:06.541 INFO:teuthology.orchestra.run.smithi083.stdout:Installing dependencies: 2024-04-07T14:23:06.541 INFO:teuthology.orchestra.run.smithi083.stdout: kernel-core x86_64 5.14.0-432.el9 baseos 15 M 2024-04-07T14:23:06.541 INFO:teuthology.orchestra.run.smithi083.stdout: kernel-modules x86_64 5.14.0-432.el9 baseos 33 M 2024-04-07T14:23:06.542 INFO:teuthology.orchestra.run.smithi083.stdout: kernel-modules-core x86_64 5.14.0-432.el9 baseos 27 M 2024-04-07T14:23:06.542 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-07T14:23:06.542 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction Summary 2024-04-07T14:23:06.542 INFO:teuthology.orchestra.run.smithi083.stdout:================================================================================ 2024-04-07T14:23:06.542 INFO:teuthology.orchestra.run.smithi083.stdout:Install 4 Packages 2024-04-07T14:23:06.542 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-07T14:23:06.542 INFO:teuthology.orchestra.run.smithi083.stdout:Total download size: 76 M 2024-04-07T14:23:06.543 INFO:teuthology.orchestra.run.smithi083.stdout:Installed size: 123 M 2024-04-07T14:23:06.543 INFO:teuthology.orchestra.run.smithi083.stdout:Downloading Packages: 2024-04-07T14:23:06.755 INFO:teuthology.orchestra.run.smithi189.stdout: Preparing : 1/1 2024-04-07T14:23:07.394 INFO:teuthology.orchestra.run.smithi083.stdout:(1/4): kernel-5.14.0-432.el9.x86_64.rpm 83 kB/s | 66 kB 00:00 2024-04-07T14:23:07.462 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : kernel-modules-core-5.14.0-432.el9.x86_64 1/4 2024-04-07T14:23:07.553 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : kernel-core-5.14.0-432.el9.x86_64 2/4 2024-04-07T14:23:10.329 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: kernel-core-5.14.0-432.el9.x86_64 2/4 2024-04-07T14:23:10.464 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : kernel-modules-5.14.0-432.el9.x86_64 3/4 2024-04-07T14:23:15.188 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: kernel-modules-5.14.0-432.el9.x86_64 3/4 2024-04-07T14:23:15.315 INFO:teuthology.orchestra.run.smithi189.stdout: Installing : kernel-5.14.0-432.el9.x86_64 4/4 2024-04-07T14:23:20.027 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: kernel-modules-core-5.14.0-432.el9.x86_64 4/4 2024-04-07T14:23:27.684 INFO:teuthology.orchestra.run.smithi083.stdout:(2/4): kernel-core-5.14.0-432.el9.x86_64.rpm 725 kB/s | 15 MB 00:21 2024-04-07T14:23:44.474 INFO:teuthology.orchestra.run.smithi083.stdout:(3/4): kernel-modules-core-5.14.0-432.el9.x86_6 759 kB/s | 27 MB 00:37 2024-04-07T14:23:49.233 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: kernel-core-5.14.0-432.el9.x86_64 4/4 2024-04-07T14:23:49.233 INFO:teuthology.orchestra.run.smithi189.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-04-07T14:23:49.233 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-04-07T14:23:49.279 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: kernel-modules-5.14.0-432.el9.x86_64 4/4 2024-04-07T14:23:49.670 INFO:teuthology.orchestra.run.smithi189.stdout: Running scriptlet: kernel-5.14.0-432.el9.x86_64 4/4 2024-04-07T14:23:49.671 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : kernel-5.14.0-432.el9.x86_64 1/4 2024-04-07T14:23:49.671 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : kernel-core-5.14.0-432.el9.x86_64 2/4 2024-04-07T14:23:49.671 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : kernel-modules-5.14.0-432.el9.x86_64 3/4 2024-04-07T14:23:50.150 INFO:teuthology.orchestra.run.smithi189.stdout: Verifying : kernel-modules-core-5.14.0-432.el9.x86_64 4/4 2024-04-07T14:23:50.150 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-04-07T14:23:50.150 INFO:teuthology.orchestra.run.smithi189.stdout:Installed: 2024-04-07T14:23:50.150 INFO:teuthology.orchestra.run.smithi189.stdout: kernel-5.14.0-432.el9.x86_64 2024-04-07T14:23:50.150 INFO:teuthology.orchestra.run.smithi189.stdout: kernel-core-5.14.0-432.el9.x86_64 2024-04-07T14:23:50.150 INFO:teuthology.orchestra.run.smithi189.stdout: kernel-modules-5.14.0-432.el9.x86_64 2024-04-07T14:23:50.151 INFO:teuthology.orchestra.run.smithi189.stdout: kernel-modules-core-5.14.0-432.el9.x86_64 2024-04-07T14:23:50.151 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-04-07T14:23:50.151 INFO:teuthology.orchestra.run.smithi189.stdout:Complete! 2024-04-07T14:23:50.491 DEBUG:teuthology.orchestra.run.smithi189:> rpm -q kernel | sort -rV | head -n 1 2024-04-07T14:23:50.593 INFO:teuthology.orchestra.run.smithi189.stdout:kernel-5.14.0-432.el9.x86_64 2024-04-07T14:23:50.593 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-432.el9.x86_64 2024-04-07T14:23:50.593 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-432.el9.x86_64 2024-04-07T14:23:50.594 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-07T14:23:50.594 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-07T14:23:50.594 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-04-07T14:23:50.594 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi189.front.sepia.ceph.com, path=None, version=distro) 2024-04-07T14:23:50.594 DEBUG:teuthology.orchestra.run.smithi189:> rpm -q kernel | sort -rV | head -n 1 2024-04-07T14:23:50.657 INFO:teuthology.orchestra.run.smithi083.stdout:(4/4): kernel-modules-5.14.0-432.el9.x86_64.rpm 778 kB/s | 33 MB 00:44 2024-04-07T14:23:50.657 INFO:teuthology.orchestra.run.smithi083.stdout:-------------------------------------------------------------------------------- 2024-04-07T14:23:50.658 INFO:teuthology.orchestra.run.smithi083.stdout:Total 1.7 MB/s | 76 MB 00:44 2024-04-07T14:23:50.675 INFO:teuthology.orchestra.run.smithi189.stdout:kernel-5.14.0-432.el9.x86_64 2024-04-07T14:23:50.675 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-432.el9.x86_64 2024-04-07T14:23:50.675 DEBUG:teuthology.orchestra.run.smithi189:> sudo rpm -qi grub2-tools 2024-04-07T14:23:50.708 INFO:teuthology.orchestra.run.smithi189.stdout:Name : grub2-tools 2024-04-07T14:23:50.709 INFO:teuthology.orchestra.run.smithi189.stdout:Epoch : 1 2024-04-07T14:23:50.709 INFO:teuthology.orchestra.run.smithi189.stdout:Version : 2.06 2024-04-07T14:23:50.709 INFO:teuthology.orchestra.run.smithi189.stdout:Release : 68.el9 2024-04-07T14:23:50.709 INFO:teuthology.orchestra.run.smithi189.stdout:Architecture: x86_64 2024-04-07T14:23:50.709 INFO:teuthology.orchestra.run.smithi189.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-04-07T14:23:50.709 INFO:teuthology.orchestra.run.smithi189.stdout:Group : Unspecified 2024-04-07T14:23:50.709 INFO:teuthology.orchestra.run.smithi189.stdout:Size : 8264254 2024-04-07T14:23:50.709 INFO:teuthology.orchestra.run.smithi189.stdout:License : GPLv3+ 2024-04-07T14:23:50.709 INFO:teuthology.orchestra.run.smithi189.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-04-07T14:23:50.709 INFO:teuthology.orchestra.run.smithi189.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-04-07T14:23:50.709 INFO:teuthology.orchestra.run.smithi189.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-04-07T14:23:50.709 INFO:teuthology.orchestra.run.smithi189.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-04-07T14:23:50.709 INFO:teuthology.orchestra.run.smithi189.stdout:Packager : builder@centos.org 2024-04-07T14:23:50.710 INFO:teuthology.orchestra.run.smithi189.stdout:Vendor : CentOS 2024-04-07T14:23:50.710 INFO:teuthology.orchestra.run.smithi189.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-07T14:23:50.710 INFO:teuthology.orchestra.run.smithi189.stdout:Summary : Support tools for GRUB. 2024-04-07T14:23:50.710 INFO:teuthology.orchestra.run.smithi189.stdout:Description : 2024-04-07T14:23:50.710 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-04-07T14:23:50.710 INFO:teuthology.orchestra.run.smithi189.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-07T14:23:50.710 INFO:teuthology.orchestra.run.smithi189.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-07T14:23:50.710 INFO:teuthology.orchestra.run.smithi189.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-07T14:23:50.710 INFO:teuthology.orchestra.run.smithi189.stdout:hardware devices. 2024-04-07T14:23:50.710 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-04-07T14:23:50.710 INFO:teuthology.orchestra.run.smithi189.stdout:This subpackage provides tools for support of all platforms. 2024-04-07T14:23:50.711 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-07T14:23:50.711 INFO:teuthology.task.kernel:Updating grub on smithi189 to boot 5.14.0-432.el9.x86_64 2024-04-07T14:23:50.711 DEBUG:teuthology.orchestra.run.smithi189:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-07T14:23:51.292 INFO:teuthology.orchestra.run.smithi189.stderr:Generating grub configuration file ... 2024-04-07T14:23:51.327 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction check 2024-04-07T14:23:51.687 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction check succeeded. 2024-04-07T14:23:51.687 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction test 2024-04-07T14:23:52.573 INFO:teuthology.orchestra.run.smithi189.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-04-07T14:23:52.605 INFO:teuthology.orchestra.run.smithi189.stderr:done 2024-04-07T14:23:52.611 DEBUG:teuthology.orchestra.run.smithi189:> mktemp 2024-04-07T14:23:52.626 INFO:teuthology.orchestra.run.smithi189.stdout:/tmp/tmp.XAJadfynav 2024-04-07T14:23:52.626 DEBUG:teuthology.orchestra.run.smithi189:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.XAJadfynav 2024-04-07T14:23:52.692 INFO:teuthology.orchestra.run.smithi083.stdout:Transaction test succeeded. 2024-04-07T14:23:52.693 INFO:teuthology.orchestra.run.smithi083.stdout:Running transaction 2024-04-07T14:23:52.695 DEBUG:teuthology.orchestra.run.smithi189:> sudo chmod 0666 /tmp/tmp.XAJadfynav 2024-04-07T14:23:52.840 DEBUG:teuthology.orchestra.remote:smithi189:/tmp/tmp.XAJadfynav is 6KB 2024-04-07T14:23:52.889 DEBUG:teuthology.orchestra.run.smithi189:> rm -fr /tmp/tmp.XAJadfynav 2024-04-07T14:23:52.903 DEBUG:teuthology.orchestra.run.smithi189:> sudo /bin/ls /boot/loader/entries || true 2024-04-07T14:23:52.968 INFO:teuthology.orchestra.run.smithi189.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-04-07T14:23:52.969 INFO:teuthology.orchestra.run.smithi189.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-04-07T14:23:52.969 INFO:teuthology.orchestra.run.smithi189.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-04-07T14:23:52.969 INFO:teuthology.orchestra.run.smithi189.stdout:15949e3c88704300882482eddfd7baae-5.14.0-432.el9.x86_64.conf 2024-04-07T14:23:52.970 DEBUG:teuthology.orchestra.run.smithi189:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-432.el9.x86_64 2024-04-07T14:23:53.146 DEBUG:teuthology.orchestra.run.smithi189:> sudo shutdown -r now 2024-04-07T14:23:55.145 INFO:teuthology.orchestra.run.smithi083.stdout: Preparing : 1/1 2024-04-07T14:23:55.767 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : kernel-modules-core-5.14.0-432.el9.x86_64 1/4 2024-04-07T14:23:55.850 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : kernel-core-5.14.0-432.el9.x86_64 2/4 2024-04-07T14:23:58.599 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: kernel-core-5.14.0-432.el9.x86_64 2/4 2024-04-07T14:23:58.738 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : kernel-modules-5.14.0-432.el9.x86_64 3/4 2024-04-07T14:24:03.437 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: kernel-modules-5.14.0-432.el9.x86_64 3/4 2024-04-07T14:24:03.539 INFO:teuthology.orchestra.run.smithi083.stdout: Installing : kernel-5.14.0-432.el9.x86_64 4/4 2024-04-07T14:24:08.253 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: kernel-modules-core-5.14.0-432.el9.x86_64 4/4 2024-04-07T14:24:23.149 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-04-07T14:24:23.150 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi189.front.sepia.ceph.com' 2024-04-07T14:24:23.150 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi189.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-07T14:24:37.624 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: kernel-core-5.14.0-432.el9.x86_64 4/4 2024-04-07T14:24:37.625 INFO:teuthology.orchestra.run.smithi083.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-04-07T14:24:37.625 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-07T14:24:37.669 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: kernel-modules-5.14.0-432.el9.x86_64 4/4 2024-04-07T14:24:38.106 INFO:teuthology.orchestra.run.smithi083.stdout: Running scriptlet: kernel-5.14.0-432.el9.x86_64 4/4 2024-04-07T14:24:38.107 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : kernel-5.14.0-432.el9.x86_64 1/4 2024-04-07T14:24:38.107 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : kernel-core-5.14.0-432.el9.x86_64 2/4 2024-04-07T14:24:38.107 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : kernel-modules-5.14.0-432.el9.x86_64 3/4 2024-04-07T14:24:38.558 INFO:teuthology.orchestra.run.smithi083.stdout: Verifying : kernel-modules-core-5.14.0-432.el9.x86_64 4/4 2024-04-07T14:24:38.558 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-07T14:24:38.559 INFO:teuthology.orchestra.run.smithi083.stdout:Installed: 2024-04-07T14:24:38.559 INFO:teuthology.orchestra.run.smithi083.stdout: kernel-5.14.0-432.el9.x86_64 2024-04-07T14:24:38.559 INFO:teuthology.orchestra.run.smithi083.stdout: kernel-core-5.14.0-432.el9.x86_64 2024-04-07T14:24:38.559 INFO:teuthology.orchestra.run.smithi083.stdout: kernel-modules-5.14.0-432.el9.x86_64 2024-04-07T14:24:38.559 INFO:teuthology.orchestra.run.smithi083.stdout: kernel-modules-core-5.14.0-432.el9.x86_64 2024-04-07T14:24:38.559 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-07T14:24:38.559 INFO:teuthology.orchestra.run.smithi083.stdout:Complete! 2024-04-07T14:24:38.912 DEBUG:teuthology.orchestra.run.smithi083:> rpm -q kernel | sort -rV | head -n 1 2024-04-07T14:24:39.011 INFO:teuthology.orchestra.run.smithi083.stdout:kernel-5.14.0-432.el9.x86_64 2024-04-07T14:24:39.011 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-432.el9.x86_64 2024-04-07T14:24:39.011 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-432.el9.x86_64 2024-04-07T14:24:39.011 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-07T14:24:39.011 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-07T14:24:39.011 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-04-07T14:24:39.011 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi083.front.sepia.ceph.com, path=None, version=distro) 2024-04-07T14:24:39.012 DEBUG:teuthology.orchestra.run.smithi083:> rpm -q kernel | sort -rV | head -n 1 2024-04-07T14:24:39.090 INFO:teuthology.orchestra.run.smithi083.stdout:kernel-5.14.0-432.el9.x86_64 2024-04-07T14:24:39.090 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-432.el9.x86_64 2024-04-07T14:24:39.091 DEBUG:teuthology.orchestra.run.smithi083:> sudo rpm -qi grub2-tools 2024-04-07T14:24:39.124 INFO:teuthology.orchestra.run.smithi083.stdout:Name : grub2-tools 2024-04-07T14:24:39.124 INFO:teuthology.orchestra.run.smithi083.stdout:Epoch : 1 2024-04-07T14:24:39.124 INFO:teuthology.orchestra.run.smithi083.stdout:Version : 2.06 2024-04-07T14:24:39.124 INFO:teuthology.orchestra.run.smithi083.stdout:Release : 68.el9 2024-04-07T14:24:39.124 INFO:teuthology.orchestra.run.smithi083.stdout:Architecture: x86_64 2024-04-07T14:24:39.124 INFO:teuthology.orchestra.run.smithi083.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-04-07T14:24:39.124 INFO:teuthology.orchestra.run.smithi083.stdout:Group : Unspecified 2024-04-07T14:24:39.124 INFO:teuthology.orchestra.run.smithi083.stdout:Size : 8264254 2024-04-07T14:24:39.125 INFO:teuthology.orchestra.run.smithi083.stdout:License : GPLv3+ 2024-04-07T14:24:39.125 INFO:teuthology.orchestra.run.smithi083.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-04-07T14:24:39.125 INFO:teuthology.orchestra.run.smithi083.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-04-07T14:24:39.125 INFO:teuthology.orchestra.run.smithi083.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-04-07T14:24:39.125 INFO:teuthology.orchestra.run.smithi083.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-04-07T14:24:39.125 INFO:teuthology.orchestra.run.smithi083.stdout:Packager : builder@centos.org 2024-04-07T14:24:39.125 INFO:teuthology.orchestra.run.smithi083.stdout:Vendor : CentOS 2024-04-07T14:24:39.125 INFO:teuthology.orchestra.run.smithi083.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-07T14:24:39.125 INFO:teuthology.orchestra.run.smithi083.stdout:Summary : Support tools for GRUB. 2024-04-07T14:24:39.125 INFO:teuthology.orchestra.run.smithi083.stdout:Description : 2024-04-07T14:24:39.125 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-07T14:24:39.125 INFO:teuthology.orchestra.run.smithi083.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-07T14:24:39.125 INFO:teuthology.orchestra.run.smithi083.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-07T14:24:39.125 INFO:teuthology.orchestra.run.smithi083.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-07T14:24:39.125 INFO:teuthology.orchestra.run.smithi083.stdout:hardware devices. 2024-04-07T14:24:39.126 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-07T14:24:39.126 INFO:teuthology.orchestra.run.smithi083.stdout:This subpackage provides tools for support of all platforms. 2024-04-07T14:24:39.126 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-07T14:24:39.126 INFO:teuthology.task.kernel:Updating grub on smithi083 to boot 5.14.0-432.el9.x86_64 2024-04-07T14:24:39.127 DEBUG:teuthology.orchestra.run.smithi083:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-07T14:24:39.798 INFO:teuthology.orchestra.run.smithi083.stderr:Generating grub configuration file ... 2024-04-07T14:24:41.185 INFO:teuthology.orchestra.run.smithi083.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-04-07T14:24:41.205 INFO:teuthology.orchestra.run.smithi083.stderr:done 2024-04-07T14:24:41.207 DEBUG:teuthology.orchestra.run.smithi083:> mktemp 2024-04-07T14:24:41.222 INFO:teuthology.orchestra.run.smithi083.stdout:/tmp/tmp.e9KwikJgIw 2024-04-07T14:24:41.222 DEBUG:teuthology.orchestra.run.smithi083:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.e9KwikJgIw 2024-04-07T14:24:41.291 DEBUG:teuthology.orchestra.run.smithi083:> sudo chmod 0666 /tmp/tmp.e9KwikJgIw 2024-04-07T14:24:41.429 DEBUG:teuthology.orchestra.remote:smithi083:/tmp/tmp.e9KwikJgIw is 6KB 2024-04-07T14:24:41.477 DEBUG:teuthology.orchestra.run.smithi083:> rm -fr /tmp/tmp.e9KwikJgIw 2024-04-07T14:24:41.490 DEBUG:teuthology.orchestra.run.smithi083:> sudo /bin/ls /boot/loader/entries || true 2024-04-07T14:24:41.502 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.189 2024-04-07T14:24:41.560 INFO:teuthology.orchestra.run.smithi083.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-04-07T14:24:41.560 INFO:teuthology.orchestra.run.smithi083.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-04-07T14:24:41.560 INFO:teuthology.orchestra.run.smithi083.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-04-07T14:24:41.560 INFO:teuthology.orchestra.run.smithi083.stdout:15949e3c88704300882482eddfd7baae-5.14.0-432.el9.x86_64.conf 2024-04-07T14:24:41.561 DEBUG:teuthology.orchestra.run.smithi083:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-432.el9.x86_64 2024-04-07T14:24:41.722 DEBUG:teuthology.orchestra.run.smithi083:> sudo shutdown -r now 2024-04-07T14:24:50.510 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi189.front.sepia.ceph.com' 2024-04-07T14:24:50.511 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi189.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-07T14:24:53.566 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.189 2024-04-07T14:25:05.571 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi189.front.sepia.ceph.com' 2024-04-07T14:25:05.572 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi189.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-07T14:25:11.725 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-04-07T14:25:11.725 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi083.front.sepia.ceph.com' 2024-04-07T14:25:11.726 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi083.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-07T14:25:23.998 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.189 2024-04-07T14:25:30.142 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.83 2024-04-07T14:25:39.006 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi189.front.sepia.ceph.com' 2024-04-07T14:25:39.007 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi189.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-07T14:25:39.143 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi083.front.sepia.ceph.com' 2024-04-07T14:25:39.144 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi083.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-07T14:25:39.449 DEBUG:teuthology.orchestra.run.smithi189:> true 2024-04-07T14:25:39.990 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi189.front.sepia.ceph.com' 2024-04-07T14:25:39.991 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-432.el9.x86_64"... 2024-04-07T14:25:39.991 DEBUG:teuthology.orchestra.run.smithi189:> uname -r 2024-04-07T14:25:40.005 INFO:teuthology.orchestra.run.smithi189.stdout:5.14.0-432.el9.x86_64 2024-04-07T14:25:40.005 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-432.el9.x86_64 vs 5.14.0-432.el9.x86_64 2024-04-07T14:25:40.005 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-07T14:25:40.005 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-07T14:25:41.006 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-04-07T14:25:41.007 DEBUG:teuthology.orchestra.run.smithi189:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-07T14:25:41.066 INFO:teuthology.orchestra.run.smithi189.stdout:ttyS1 2024-04-07T14:25:41.107 DEBUG:teuthology.parallel:result is None 2024-04-07T14:25:42.206 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.83 2024-04-07T14:25:54.211 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi083.front.sepia.ceph.com' 2024-04-07T14:25:54.212 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi083.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-07T14:26:26.074 DEBUG:teuthology.orchestra.run.smithi083:> true 2024-04-07T14:26:26.473 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi083.front.sepia.ceph.com' 2024-04-07T14:26:26.474 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-432.el9.x86_64"... 2024-04-07T14:26:26.474 DEBUG:teuthology.orchestra.run.smithi083:> uname -r 2024-04-07T14:26:26.488 INFO:teuthology.orchestra.run.smithi083.stdout:5.14.0-432.el9.x86_64 2024-04-07T14:26:26.488 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-432.el9.x86_64 vs 5.14.0-432.el9.x86_64 2024-04-07T14:26:26.488 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-07T14:26:26.488 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-07T14:26:27.489 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-04-07T14:26:27.489 DEBUG:teuthology.orchestra.run.smithi083:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-07T14:26:27.595 INFO:teuthology.orchestra.run.smithi083.stdout:ttyS1 2024-04-07T14:26:27.627 DEBUG:teuthology.parallel:result is None 2024-04-07T14:26:27.628 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-07T14:26:27.635 INFO:teuthology.task.internal:Creating test directory... 2024-04-07T14:26:27.635 DEBUG:teuthology.orchestra.run.smithi083:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-07T14:26:27.639 DEBUG:teuthology.orchestra.run.smithi189:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-07T14:26:27.659 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-07T14:26:27.665 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-07T14:26:27.672 INFO:teuthology.task.internal:Creating archive directory... 2024-04-07T14:26:27.673 DEBUG:teuthology.orchestra.run.smithi083:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-07T14:26:27.697 DEBUG:teuthology.orchestra.run.smithi189:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-07T14:26:27.768 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-07T14:26:27.775 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-07T14:26:27.775 DEBUG:teuthology.orchestra.run.smithi083:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-07T14:26:27.777 DEBUG:teuthology.orchestra.run.smithi189:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-07T14:26:27.813 INFO:teuthology.orchestra.run.smithi189.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-07T14:26:27.827 INFO:teuthology.orchestra.run.smithi189.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-07T14:26:27.880 INFO:teuthology.orchestra.run.smithi083.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-07T14:26:27.900 INFO:teuthology.orchestra.run.smithi083.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-07T14:26:27.902 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-07T14:26:27.908 INFO:teuthology.task.internal:Configuring sudo... 2024-04-07T14:26:27.908 DEBUG:teuthology.orchestra.run.smithi083:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-07T14:26:27.911 DEBUG:teuthology.orchestra.run.smithi189:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-07T14:26:27.956 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-07T14:26:27.965 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-07T14:26:27.965 DEBUG:teuthology.orchestra.run.smithi083:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-07T14:26:27.968 DEBUG:teuthology.orchestra.run.smithi189:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-07T14:26:27.997 DEBUG:teuthology.orchestra.run.smithi083:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-07T14:26:28.082 DEBUG:teuthology.orchestra.run.smithi083:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-07T14:26:28.161 DEBUG:teuthology.orchestra.run.smithi083:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-07T14:26:28.218 DEBUG:teuthology.orchestra.run.smithi083:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-07T14:26:28.272 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-04-07T14:26:28.272 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-07T14:26:28.318 DEBUG:teuthology.orchestra.run.smithi189:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-07T14:26:28.361 DEBUG:teuthology.orchestra.run.smithi189:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-07T14:26:28.421 DEBUG:teuthology.orchestra.run.smithi189:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-07T14:26:28.463 DEBUG:teuthology.orchestra.run.smithi189:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-07T14:26:28.506 DEBUG:teuthology.orchestra.run.smithi189:> set -ex 2024-04-07T14:26:28.507 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-07T14:26:28.575 DEBUG:teuthology.orchestra.run.smithi083:> sudo service rsyslog restart 2024-04-07T14:26:28.577 DEBUG:teuthology.orchestra.run.smithi189:> sudo service rsyslog restart 2024-04-07T14:26:28.623 INFO:teuthology.orchestra.run.smithi083.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-07T14:26:28.663 INFO:teuthology.orchestra.run.smithi189.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-07T14:26:28.871 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-07T14:26:28.885 INFO:teuthology.task.internal:Starting timer... 2024-04-07T14:26:28.885 INFO:teuthology.run_tasks:Running task pcp... 2024-04-07T14:26:28.895 INFO:teuthology.run_tasks:Running task selinux... 2024-04-07T14:26:28.903 DEBUG:teuthology.orchestra.run.smithi083:> sudo service auditd rotate 2024-04-07T14:26:29.021 INFO:teuthology.orchestra.run.smithi083.stdout:Rotating logs: 2024-04-07T14:26:29.023 DEBUG:teuthology.orchestra.run.smithi189:> sudo service auditd rotate 2024-04-07T14:26:29.073 INFO:teuthology.orchestra.run.smithi189.stdout:Rotating logs: 2024-04-07T14:26:29.075 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-07T14:26:29.075 DEBUG:teuthology.orchestra.run.smithi083:> /usr/sbin/getenforce 2024-04-07T14:26:29.146 INFO:teuthology.orchestra.run.smithi083.stdout:Permissive 2024-04-07T14:26:29.146 DEBUG:teuthology.orchestra.run.smithi189:> /usr/sbin/getenforce 2024-04-07T14:26:29.163 INFO:teuthology.orchestra.run.smithi189.stdout:Permissive 2024-04-07T14:26:29.163 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi083.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi189.front.sepia.ceph.com': 'permissive'} 2024-04-07T14:26:29.164 DEBUG:teuthology.orchestra.run.smithi083:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-07T14:26:29.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-07T14:26:29.192 DEBUG:teuthology.orchestra.run.smithi189:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-07T14:26:29.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-07T14:26:29.231 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-07T14:26:29.231 DEBUG:teuthology.orchestra.run.smithi083:> sudo /usr/sbin/setenforce permissive 2024-04-07T14:26:29.264 DEBUG:teuthology.orchestra.run.smithi189:> sudo /usr/sbin/setenforce permissive 2024-04-07T14:26:29.306 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-07T14:26:29.317 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-07T14:26:29.333 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-07T14:26:29.334 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi083.front.sepia.ceph.com,smithi189.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-07T14:30:58.115 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi083.front.sepia.ceph.com'), Remote(name='ubuntu@smithi189.front.sepia.ceph.com')] 2024-04-07T14:30:58.116 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi083.front.sepia.ceph.com' 2024-04-07T14:30:58.117 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi083.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-07T14:30:58.209 DEBUG:teuthology.orchestra.run.smithi083:> true 2024-04-07T14:30:58.291 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi083.front.sepia.ceph.com' 2024-04-07T14:30:58.291 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi189.front.sepia.ceph.com' 2024-04-07T14:30:58.292 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi189.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-07T14:30:58.382 DEBUG:teuthology.orchestra.run.smithi189:> true 2024-04-07T14:30:58.461 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi189.front.sepia.ceph.com' 2024-04-07T14:30:58.462 INFO:teuthology.run_tasks:Running task clock... 2024-04-07T14:30:58.473 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-04-07T14:30:58.473 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-07T14:30:58.474 DEBUG:teuthology.orchestra.run.smithi083:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-07T14:30:58.476 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-07T14:30:58.476 DEBUG:teuthology.orchestra.run.smithi189:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-07T14:30:58.506 INFO:teuthology.orchestra.run.smithi083.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-07T14:30:58.523 INFO:teuthology.orchestra.run.smithi083.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-07T14:30:58.534 INFO:teuthology.orchestra.run.smithi189.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-07T14:30:58.551 INFO:teuthology.orchestra.run.smithi189.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-07T14:30:58.552 INFO:teuthology.orchestra.run.smithi083.stderr:sudo: ntpd: command not found 2024-04-07T14:30:58.566 INFO:teuthology.orchestra.run.smithi083.stdout:506 Cannot talk to daemon 2024-04-07T14:30:58.580 INFO:teuthology.orchestra.run.smithi189.stderr:sudo: ntpd: command not found 2024-04-07T14:30:58.582 INFO:teuthology.orchestra.run.smithi083.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-07T14:30:58.594 INFO:teuthology.orchestra.run.smithi189.stdout:506 Cannot talk to daemon 2024-04-07T14:30:58.598 INFO:teuthology.orchestra.run.smithi083.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-07T14:30:58.610 INFO:teuthology.orchestra.run.smithi189.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-07T14:30:58.627 INFO:teuthology.orchestra.run.smithi189.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-07T14:30:58.650 INFO:teuthology.orchestra.run.smithi083.stderr:bash: line 1: ntpq: command not found 2024-04-07T14:30:58.653 INFO:teuthology.orchestra.run.smithi083.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-07T14:30:58.653 INFO:teuthology.orchestra.run.smithi083.stdout:=============================================================================== 2024-04-07T14:30:58.653 INFO:teuthology.orchestra.run.smithi083.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-07T14:30:58.653 INFO:teuthology.orchestra.run.smithi083.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-07T14:30:58.654 INFO:teuthology.orchestra.run.smithi083.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-07T14:30:58.654 INFO:teuthology.orchestra.run.smithi083.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-07T14:30:58.681 INFO:teuthology.orchestra.run.smithi189.stderr:bash: line 1: ntpq: command not found 2024-04-07T14:30:58.684 INFO:teuthology.orchestra.run.smithi189.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-07T14:30:58.684 INFO:teuthology.orchestra.run.smithi189.stdout:=============================================================================== 2024-04-07T14:30:58.684 INFO:teuthology.orchestra.run.smithi189.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-07T14:30:58.684 INFO:teuthology.orchestra.run.smithi189.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-07T14:30:58.684 INFO:teuthology.orchestra.run.smithi189.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-07T14:30:58.684 INFO:teuthology.orchestra.run.smithi189.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-07T14:30:58.685 INFO:teuthology.run_tasks:Running task cephadm... 2024-04-07T14:30:58.785 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'conf': {'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1, 'mgr/cephadm/use_agent': True}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'e5c885fe95ee982ff7b45214daa61b1a6d09d57d'} 2024-04-07T14:30:58.785 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-04-07T14:30:58.786 INFO:tasks.cephadm:Cluster fsid is 7388281a-f4eb-11ee-b649-cb9ed24678a4 2024-04-07T14:30:58.786 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-04-07T14:30:58.786 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.83', 'mon.c': '[v2:172.21.15.83:3301,v1:172.21.15.83:6790]', 'mon.b': '172.21.15.189'} 2024-04-07T14:30:58.786 INFO:tasks.cephadm:First mon is mon.a on smithi083 2024-04-07T14:30:58.786 INFO:tasks.cephadm:First mgr is y 2024-04-07T14:30:58.786 INFO:tasks.cephadm:Normalizing hostnames... 2024-04-07T14:30:58.786 DEBUG:teuthology.orchestra.run.smithi083:> sudo hostname $(hostname -s) 2024-04-07T14:30:58.814 DEBUG:teuthology.orchestra.run.smithi189:> sudo hostname $(hostname -s) 2024-04-07T14:30:58.842 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-04-07T14:30:58.843 DEBUG:teuthology.orchestra.run.smithi083:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-07T14:30:59.125 INFO:teuthology.orchestra.run.smithi083.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Apr 7 14:30 /home/ubuntu/cephtest/cephadm 2024-04-07T14:30:59.125 DEBUG:teuthology.orchestra.run.smithi189:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-07T14:30:59.298 INFO:teuthology.orchestra.run.smithi189.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Apr 7 14:30 /home/ubuntu/cephtest/cephadm 2024-04-07T14:30:59.298 DEBUG:teuthology.orchestra.run.smithi083:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-07T14:30:59.314 DEBUG:teuthology.orchestra.run.smithi189:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-07T14:30:59.340 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-04-07T14:30:59.341 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-04-07T14:30:59.356 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-04-07T14:31:00.207 INFO:teuthology.orchestra.run.smithi189.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-07T14:31:00.269 INFO:teuthology.orchestra.run.smithi083.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-07T14:31:21.920 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-04-07T14:31:21.920 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-04-07T14:31:21.920 INFO:teuthology.orchestra.run.smithi083.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-04-07T14:31:21.920 INFO:teuthology.orchestra.run.smithi083.stdout: "repo_digests": [ 2024-04-07T14:31:21.920 INFO:teuthology.orchestra.run.smithi083.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-04-07T14:31:21.920 INFO:teuthology.orchestra.run.smithi083.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-04-07T14:31:21.921 INFO:teuthology.orchestra.run.smithi083.stdout: ] 2024-04-07T14:31:21.921 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-04-07T14:31:21.958 INFO:teuthology.orchestra.run.smithi189.stdout:{ 2024-04-07T14:31:21.959 INFO:teuthology.orchestra.run.smithi189.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-04-07T14:31:21.959 INFO:teuthology.orchestra.run.smithi189.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-04-07T14:31:21.959 INFO:teuthology.orchestra.run.smithi189.stdout: "repo_digests": [ 2024-04-07T14:31:21.959 INFO:teuthology.orchestra.run.smithi189.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-04-07T14:31:21.959 INFO:teuthology.orchestra.run.smithi189.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-04-07T14:31:21.959 INFO:teuthology.orchestra.run.smithi189.stdout: ] 2024-04-07T14:31:21.959 INFO:teuthology.orchestra.run.smithi189.stdout:} 2024-04-07T14:31:21.977 DEBUG:teuthology.orchestra.run.smithi083:> sudo mkdir -p /etc/ceph 2024-04-07T14:31:22.006 DEBUG:teuthology.orchestra.run.smithi189:> sudo mkdir -p /etc/ceph 2024-04-07T14:31:22.036 DEBUG:teuthology.orchestra.run.smithi083:> sudo chmod 777 /etc/ceph 2024-04-07T14:31:22.073 DEBUG:teuthology.orchestra.run.smithi189:> sudo chmod 777 /etc/ceph 2024-04-07T14:31:22.102 INFO:tasks.cephadm:Writing seed config... 2024-04-07T14:31:22.103 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2024-04-07T14:31:22.104 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-04-07T14:31:22.104 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-04-07T14:31:22.104 INFO:tasks.cephadm: override: [mgr] mgr/cephadm/use_agent = True 2024-04-07T14:31:22.104 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-04-07T14:31:22.104 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-04-07T14:31:22.104 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-04-07T14:31:22.104 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-04-07T14:31:22.104 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-04-07T14:31:22.104 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-04-07T14:31:22.105 DEBUG:teuthology.orchestra.run.smithi083:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-04-07T14:31:22.127 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 7388281a-f4eb-11ee-b649-cb9ed24678a4 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = true bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 mgr/cephadm/use_agent = True [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-04-07T14:31:22.127 DEBUG:teuthology.orchestra.run.smithi083:mon.a> sudo journalctl -f -n 0 -u ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@mon.a.service 2024-04-07T14:31:22.170 DEBUG:teuthology.orchestra.run.smithi083:mgr.y> sudo journalctl -f -n 0 -u ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@mgr.y.service 2024-04-07T14:31:22.212 INFO:tasks.cephadm:Bootstrapping... 2024-04-07T14:31:22.212 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.83 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-04-07T14:31:22.406 INFO:teuthology.orchestra.run.smithi083.stderr:-------------------------------------------------------------------------------- 2024-04-07T14:31:22.406 INFO:teuthology.orchestra.run.smithi083.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '7388281a-f4eb-11ee-b649-cb9ed24678a4', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.83', '--skip-admin-label'] 2024-04-07T14:31:22.429 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: 4.9.4-dev 2024-04-07T14:31:22.431 INFO:teuthology.orchestra.run.smithi083.stderr:Verifying podman|docker is present... 2024-04-07T14:31:22.453 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: 4.9.4-dev 2024-04-07T14:31:22.456 INFO:teuthology.orchestra.run.smithi083.stderr:Verifying lvm2 is present... 2024-04-07T14:31:22.456 INFO:teuthology.orchestra.run.smithi083.stderr:Verifying time synchronization is in place... 2024-04-07T14:31:22.463 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-07T14:31:22.469 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: inactive 2024-04-07T14:31:22.475 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: enabled 2024-04-07T14:31:22.481 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: active 2024-04-07T14:31:22.482 INFO:teuthology.orchestra.run.smithi083.stderr:Unit chronyd.service is enabled and running 2024-04-07T14:31:22.482 INFO:teuthology.orchestra.run.smithi083.stderr:Repeating the final host check... 2024-04-07T14:31:22.502 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: 4.9.4-dev 2024-04-07T14:31:22.504 INFO:teuthology.orchestra.run.smithi083.stderr:podman (/bin/podman) version 4.9.4 is present 2024-04-07T14:31:22.504 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl is present 2024-04-07T14:31:22.505 INFO:teuthology.orchestra.run.smithi083.stderr:lvcreate is present 2024-04-07T14:31:22.510 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-07T14:31:22.516 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: inactive 2024-04-07T14:31:22.523 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: enabled 2024-04-07T14:31:22.528 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: active 2024-04-07T14:31:22.529 INFO:teuthology.orchestra.run.smithi083.stderr:Unit chronyd.service is enabled and running 2024-04-07T14:31:22.529 INFO:teuthology.orchestra.run.smithi083.stderr:Host looks OK 2024-04-07T14:31:22.529 INFO:teuthology.orchestra.run.smithi083.stderr:Cluster fsid: 7388281a-f4eb-11ee-b649-cb9ed24678a4 2024-04-07T14:31:22.529 INFO:teuthology.orchestra.run.smithi083.stderr:Acquiring lock 140077030772064 on /run/cephadm/7388281a-f4eb-11ee-b649-cb9ed24678a4.lock 2024-04-07T14:31:22.530 INFO:teuthology.orchestra.run.smithi083.stderr:Lock 140077030772064 acquired on /run/cephadm/7388281a-f4eb-11ee-b649-cb9ed24678a4.lock 2024-04-07T14:31:22.530 INFO:teuthology.orchestra.run.smithi083.stderr:Verifying IP 172.21.15.83 port 3300 ... 2024-04-07T14:31:22.530 INFO:teuthology.orchestra.run.smithi083.stderr:Verifying IP 172.21.15.83 port 6789 ... 2024-04-07T14:31:22.530 INFO:teuthology.orchestra.run.smithi083.stderr:Base mon IP is 172.21.15.83, final addrv is [v2:172.21.15.83:3300,v1:172.21.15.83:6789] 2024-04-07T14:31:22.533 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.83 metric 100 2024-04-07T14:31:22.533 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.83 metric 100 2024-04-07T14:31:22.535 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-04-07T14:31:22.536 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2024-04-07T14:31:22.536 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 1024 expires 1786sec hoplimit 64 pref medium 2024-04-07T14:31:22.538 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-04-07T14:31:22.538 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-04-07T14:31:22.538 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-07T14:31:22.538 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-04-07T14:31:22.538 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: inet6 fe80::ec4:7aff:fe8f:ce91/64 scope link 2024-04-07T14:31:22.539 INFO:teuthology.orchestra.run.smithi083.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-07T14:31:22.539 INFO:teuthology.orchestra.run.smithi083.stderr:Mon IP `172.21.15.83` is in CIDR network `172.21.0.0/20` 2024-04-07T14:31:22.539 INFO:teuthology.orchestra.run.smithi083.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-04-07T14:31:22.540 INFO:teuthology.orchestra.run.smithi083.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-07T14:31:22.639 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-04-07T14:31:23.041 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Getting image source signatures 2024-04-07T14:31:23.041 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-04-07T14:31:23.041 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-04-07T14:31:23.041 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-04-07T14:31:23.041 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-04-07T14:31:23.041 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-04-07T14:31:23.041 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-04-07T14:31:23.191 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: Writing manifest to image destination 2024-04-07T14:31:23.343 INFO:teuthology.orchestra.run.smithi083.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-04-07T14:31:23.751 INFO:teuthology.orchestra.run.smithi083.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-04-07T14:31:24.242 INFO:teuthology.orchestra.run.smithi083.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-04-07T14:31:24.242 INFO:teuthology.orchestra.run.smithi083.stderr:Extracting ceph user uid/gid from container image... 2024-04-07T14:31:24.659 INFO:teuthology.orchestra.run.smithi083.stderr:stat: 167 167 2024-04-07T14:31:25.106 INFO:teuthology.orchestra.run.smithi083.stderr:Creating initial keys... 2024-04-07T14:31:25.464 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-authtool: AQA9rhJmh9mYFhAA8EPb0iogTJdTK6UfIRo+SQ== 2024-04-07T14:31:26.411 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-authtool: AQA+rhJmGgf1ExAAK8Atuzvs/DnqsZsXI53m4A== 2024-04-07T14:31:27.341 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph-authtool: AQA/rhJmmoPXEBAAHr5kKfrIgYZaOarw3pUeoA== 2024-04-07T14:31:27.773 INFO:teuthology.orchestra.run.smithi083.stderr:Creating initial monmap... 2024-04-07T14:31:28.206 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-07T14:31:28.206 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-04-07T14:31:28.206 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 7388281a-f4eb-11ee-b649-cb9ed24678a4 2024-04-07T14:31:28.207 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-07T14:31:28.661 INFO:teuthology.orchestra.run.smithi083.stderr:monmaptool for a [v2:172.21.15.83:3300,v1:172.21.15.83:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-07T14:31:28.661 INFO:teuthology.orchestra.run.smithi083.stderr:setting min_mon_release = octopus 2024-04-07T14:31:28.661 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/monmaptool: set fsid to 7388281a-f4eb-11ee-b649-cb9ed24678a4 2024-04-07T14:31:28.661 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-07T14:31:28.661 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-04-07T14:31:28.662 INFO:teuthology.orchestra.run.smithi083.stderr:Creating mon... 2024-04-07T14:31:29.551 INFO:teuthology.orchestra.run.smithi083.stderr:create mon.a on 2024-04-07T14:31:29.772 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-04-07T14:31:29.914 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4.target → /etc/systemd/system/ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4.target. 2024-04-07T14:31:29.915 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4.target → /etc/systemd/system/ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4.target. 2024-04-07T14:31:30.208 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Failed to reset failed state of unit ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@mon.a.service: Unit ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@mon.a.service not loaded. 2024-04-07T14:31:30.215 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Created symlink /etc/systemd/system/ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4.target.wants/ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@mon.a.service → /etc/systemd/system/ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@.service. 2024-04-07T14:31:30.844 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:30 smithi083 ceph-mon[24452]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-07T14:31:30.856 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: disabled 2024-04-07T14:31:30.863 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: inactive 2024-04-07T14:31:30.863 INFO:teuthology.orchestra.run.smithi083.stderr:firewalld.service is not enabled 2024-04-07T14:31:30.864 INFO:teuthology.orchestra.run.smithi083.stderr:Not possible to enable service . firewalld.service is not available 2024-04-07T14:31:30.864 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for mon to start... 2024-04-07T14:31:30.864 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for mon... 2024-04-07T14:31:31.099 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:30 smithi083 bash[24433]: 3d4d6ff8353919f2ca4d2ee7a77988fe5be35e0562cbd56983b71f3c7703a827 2024-04-07T14:31:31.099 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:30 smithi083 systemd[1]: Started Ceph mon.a for 7388281a-f4eb-11ee-b649-cb9ed24678a4. 2024-04-07T14:31:31.376 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: cluster: 2024-04-07T14:31:31.377 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: id: 7388281a-f4eb-11ee-b649-cb9ed24678a4 2024-04-07T14:31:31.377 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-04-07T14:31:31.377 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-04-07T14:31:31.377 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: services: 2024-04-07T14:31:31.377 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon: 1 daemons, quorum a (age 0.567086s) 2024-04-07T14:31:31.377 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mgr: no daemons active 2024-04-07T14:31:31.377 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-04-07T14:31:31.377 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-04-07T14:31:31.377 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: data: 2024-04-07T14:31:31.377 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-04-07T14:31:31.377 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-04-07T14:31:31.377 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-04-07T14:31:31.377 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: pgs: 2024-04-07T14:31:31.377 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-04-07T14:31:31.986 INFO:teuthology.orchestra.run.smithi083.stderr:mon is available 2024-04-07T14:31:31.987 INFO:teuthology.orchestra.run.smithi083.stderr:Assimilating anything we can from ceph.conf... 2024-04-07T14:31:32.084 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:31 smithi083 ceph-mon[24452]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-07T14:31:32.085 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:31 smithi083 ceph-mon[24452]: monmap e1: 1 mons at {a=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0]} 2024-04-07T14:31:32.085 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:31 smithi083 ceph-mon[24452]: fsmap 2024-04-07T14:31:32.085 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:31 smithi083 ceph-mon[24452]: osdmap e1: 0 total, 0 up, 0 in 2024-04-07T14:31:32.085 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:31 smithi083 ceph-mon[24452]: mgrmap e1: no daemons active 2024-04-07T14:31:32.085 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:31 smithi083 ceph-mon[24452]: from='client.? 172.21.15.83:0/40251845' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-07T14:31:32.482 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-04-07T14:31:32.483 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: [global] 2024-04-07T14:31:32.483 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: fsid = 7388281a-f4eb-11ee-b649-cb9ed24678a4 2024-04-07T14:31:32.483 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.83:3300,v1:172.21.15.83:6789] 2024-04-07T14:31:32.483 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-04-07T14:31:32.483 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-04-07T14:31:32.483 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-04-07T14:31:32.483 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-04-07T14:31:32.483 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-04-07T14:31:32.483 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: [mgr] 2024-04-07T14:31:32.483 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mgr/cephadm/use_agent = True 2024-04-07T14:31:32.483 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-04-07T14:31:32.483 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-04-07T14:31:32.484 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: [osd] 2024-04-07T14:31:32.484 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-04-07T14:31:32.484 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-04-07T14:31:32.926 INFO:teuthology.orchestra.run.smithi083.stderr:Generating new minimal ceph.conf... 2024-04-07T14:31:33.123 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:32 smithi083 ceph-mon[24452]: from='client.? 172.21.15.83:0/2972636820' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-07T14:31:33.123 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:32 smithi083 ceph-mon[24452]: from='client.? 172.21.15.83:0/2972636820' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-07T14:31:33.924 INFO:teuthology.orchestra.run.smithi083.stderr:Restarting the monitor... 2024-04-07T14:31:34.109 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:33 smithi083 ceph-mon[24452]: from='client.? 172.21.15.83:0/2829256036' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:31:34.109 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:33 smithi083 systemd[1]: Stopping Ceph mon.a for 7388281a-f4eb-11ee-b649-cb9ed24678a4... 2024-04-07T14:31:34.109 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:33 smithi083 bash[24673]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mon.a" found: no such container 2024-04-07T14:31:34.109 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:34 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mon-a[24448]: 2024-04-07T14:31:34.057+0000 7f3652d11700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-07T14:31:34.109 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:34 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mon-a[24448]: 2024-04-07T14:31:34.057+0000 7f3652d11700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-04-07T14:31:34.375 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:34 smithi083 podman[24684]: 2024-04-07 14:31:34.109901338 +0000 UTC m=+0.133969577 container died 3d4d6ff8353919f2ca4d2ee7a77988fe5be35e0562cbd56983b71f3c7703a827 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mon-a, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, maintainer=Guillaume Abrioux , name=centos-stream, distribution-scope=public, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, vendor=Red Hat, Inc., architecture=x86_64, io.k8s.display-name=CentOS Stream 8, version=8, ceph=True, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0) 2024-04-07T14:31:34.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:34 smithi083 podman[24684]: 2024-04-07 14:31:34.475419988 +0000 UTC m=+0.499488220 container cleanup 3d4d6ff8353919f2ca4d2ee7a77988fe5be35e0562cbd56983b71f3c7703a827 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mon-a, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, name=centos-stream, distribution-scope=public, com.redhat.component=centos-stream-container, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, GIT_CLEAN=True, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream) 2024-04-07T14:31:34.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:34 smithi083 bash[24684]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mon-a 2024-04-07T14:31:34.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:34 smithi083 bash[24712]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mon.a" found: no such container 2024-04-07T14:31:34.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:34 smithi083 podman[24697]: 2024-04-07 14:31:34.768233225 +0000 UTC m=+0.653500498 container remove 3d4d6ff8353919f2ca4d2ee7a77988fe5be35e0562cbd56983b71f3c7703a827 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mon-a, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, vendor=Red Hat, Inc., name=centos-stream, distribution-scope=public, ceph=True, version=8, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD) 2024-04-07T14:31:34.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:34 smithi083 systemd[1]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@mon.a.service: Deactivated successfully. 2024-04-07T14:31:34.967 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:34 smithi083 systemd[1]: Stopped Ceph mon.a for 7388281a-f4eb-11ee-b649-cb9ed24678a4. 2024-04-07T14:31:34.968 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:34 smithi083 systemd[1]: Starting Ceph mon.a for 7388281a-f4eb-11ee-b649-cb9ed24678a4... 2024-04-07T14:31:35.228 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:34 smithi083 podman[24790]: 2024-04-07 14:31:34.968856329 +0000 UTC m=+0.088431654 container create 045b78f11b8e0a839a0de89ffde1b01200a6e2b3f8070084bb844c2a9e030bc8 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mon-a, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, com.redhat.component=centos-stream-container, vcs-type=git, io.buildah.version=1.19.8, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., name=centos-stream, release=754, RELEASE=HEAD, architecture=x86_64, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8) 2024-04-07T14:31:35.228 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:34 smithi083 podman[24790]: 2024-04-07 14:31:34.899062229 +0000 UTC m=+0.018637553 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-04-07T14:31:35.228 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 podman[24790]: 2024-04-07 14:31:35.146338703 +0000 UTC m=+0.265914029 container init 045b78f11b8e0a839a0de89ffde1b01200a6e2b3f8070084bb844c2a9e030bc8 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mon-a, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, version=8, distribution-scope=public, vcs-type=git, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=HEAD, ceph=True, release=754, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container) 2024-04-07T14:31:35.229 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 podman[24790]: 2024-04-07 14:31:35.14957189 +0000 UTC m=+0.269147217 container start 045b78f11b8e0a839a0de89ffde1b01200a6e2b3f8070084bb844c2a9e030bc8 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mon-a, vendor=Red Hat, Inc., RELEASE=HEAD, architecture=x86_64, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, io.buildah.version=1.19.8, GIT_CLEAN=True, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, distribution-scope=public, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870) 2024-04-07T14:31:35.229 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: set uid:gid to 167:167 (ceph:ceph) 2024-04-07T14:31:35.229 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-04-07T14:31:35.229 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: pidfile_write: ignore empty --pid-file 2024-04-07T14:31:35.229 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: load: jerasure load: lrc 2024-04-07T14:31:35.229 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: RocksDB version: 6.15.5 2024-04-07T14:31:35.229 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-07T14:31:35.229 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Compile date Apr 18 2022 2024-04-07T14:31:35.229 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: DB SUMMARY 2024-04-07T14:31:35.229 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: DB Session ID: RU8PJFBVQS1QLXO4979J 2024-04-07T14:31:35.229 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: CURRENT file: CURRENT 2024-04-07T14:31:35.229 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: IDENTITY file: IDENTITY 2024-04-07T14:31:35.229 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-04-07T14:31:35.229 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-04-07T14:31:35.230 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 82629 ; 2024-04-07T14:31:35.230 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.error_if_exists: 0 2024-04-07T14:31:35.230 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.create_if_missing: 0 2024-04-07T14:31:35.230 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.paranoid_checks: 1 2024-04-07T14:31:35.230 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-07T14:31:35.230 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.env: 0x5641ab2c6860 2024-04-07T14:31:35.230 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.fs: Posix File System 2024-04-07T14:31:35.230 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.info_log: 0x5641ac389dc0 2024-04-07T14:31:35.230 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.max_file_opening_threads: 16 2024-04-07T14:31:35.230 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.statistics: (nil) 2024-04-07T14:31:35.230 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.use_fsync: 0 2024-04-07T14:31:35.230 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.max_log_file_size: 0 2024-04-07T14:31:35.230 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-07T14:31:35.231 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-07T14:31:35.231 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.keep_log_file_num: 1000 2024-04-07T14:31:35.231 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.recycle_log_file_num: 0 2024-04-07T14:31:35.232 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.allow_fallocate: 1 2024-04-07T14:31:35.232 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.allow_mmap_reads: 0 2024-04-07T14:31:35.232 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.allow_mmap_writes: 0 2024-04-07T14:31:35.232 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.use_direct_reads: 0 2024-04-07T14:31:35.232 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-07T14:31:35.232 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.create_missing_column_families: 0 2024-04-07T14:31:35.232 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.db_log_dir: 2024-04-07T14:31:35.232 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-04-07T14:31:35.232 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-07T14:31:35.232 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-07T14:31:35.232 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-07T14:31:35.232 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-07T14:31:35.232 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-07T14:31:35.232 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-07T14:31:35.233 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.advise_random_on_open: 1 2024-04-07T14:31:35.233 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.db_write_buffer_size: 0 2024-04-07T14:31:35.233 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.write_buffer_manager: 0x5641ac47a2a0 2024-04-07T14:31:35.233 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-07T14:31:35.233 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-07T14:31:35.233 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-07T14:31:35.233 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-07T14:31:35.233 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.rate_limiter: (nil) 2024-04-07T14:31:35.233 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-07T14:31:35.233 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.wal_recovery_mode: 2 2024-04-07T14:31:35.233 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.enable_thread_tracking: 0 2024-04-07T14:31:35.233 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.enable_pipelined_write: 0 2024-04-07T14:31:35.233 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.unordered_write: 0 2024-04-07T14:31:35.233 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-07T14:31:35.233 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-07T14:31:35.234 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-07T14:31:35.234 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-07T14:31:35.234 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.row_cache: None 2024-04-07T14:31:35.234 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.wal_filter: None 2024-04-07T14:31:35.234 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-07T14:31:35.234 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.allow_ingest_behind: 0 2024-04-07T14:31:35.234 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.preserve_deletes: 0 2024-04-07T14:31:35.234 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.two_write_queues: 0 2024-04-07T14:31:35.234 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.manual_wal_flush: 0 2024-04-07T14:31:35.234 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.atomic_flush: 0 2024-04-07T14:31:35.234 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-07T14:31:35.234 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-07T14:31:35.234 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-07T14:31:35.235 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.log_readahead_size: 0 2024-04-07T14:31:35.235 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-07T14:31:35.235 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.best_efforts_recovery: 0 2024-04-07T14:31:35.235 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-07T14:31:35.235 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-07T14:31:35.235 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.allow_data_in_errors: 0 2024-04-07T14:31:35.235 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.db_host_id: __hostname__ 2024-04-07T14:31:35.235 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.max_background_jobs: 2 2024-04-07T14:31:35.235 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.max_background_compactions: -1 2024-04-07T14:31:35.235 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.max_subcompactions: 1 2024-04-07T14:31:35.235 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-07T14:31:35.235 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-07T14:31:35.235 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-07T14:31:35.235 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.max_total_wal_size: 0 2024-04-07T14:31:35.235 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-07T14:31:35.236 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-07T14:31:35.236 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-07T14:31:35.236 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-07T14:31:35.236 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.max_open_files: -1 2024-04-07T14:31:35.236 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.bytes_per_sync: 0 2024-04-07T14:31:35.236 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-07T14:31:35.236 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-07T14:31:35.236 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.compaction_readahead_size: 0 2024-04-07T14:31:35.236 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.max_background_flushes: -1 2024-04-07T14:31:35.236 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Compression algorithms supported: 2024-04-07T14:31:35.236 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-07T14:31:35.236 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: kZSTD supported: 0 2024-04-07T14:31:35.236 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: kXpressCompression supported: 0 2024-04-07T14:31:35.236 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: kLZ4HCCompression supported: 1 2024-04-07T14:31:35.236 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: kLZ4Compression supported: 1 2024-04-07T14:31:35.237 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: kBZip2Compression supported: 0 2024-04-07T14:31:35.237 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: kZlibCompression supported: 1 2024-04-07T14:31:35.237 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: kSnappyCompression supported: 1 2024-04-07T14:31:35.237 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-07T14:31:35.237 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-04-07T14:31:35.237 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-04-07T14:31:35.237 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-07T14:31:35.237 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.merge_operator: 2024-04-07T14:31:35.237 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.compaction_filter: None 2024-04-07T14:31:35.237 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.compaction_filter_factory: None 2024-04-07T14:31:35.237 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.sst_partitioner_factory: None 2024-04-07T14:31:35.237 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-07T14:31:35.237 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-07T14:31:35.237 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5641ac355d00) 2024-04-07T14:31:35.237 INFO:journalctl@ceph.mon.a.smithi083.stdout: cache_index_and_filter_blocks: 1 2024-04-07T14:31:35.237 INFO:journalctl@ceph.mon.a.smithi083.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-07T14:31:35.238 INFO:journalctl@ceph.mon.a.smithi083.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-07T14:31:35.238 INFO:journalctl@ceph.mon.a.smithi083.stdout: pin_top_level_index_and_filter: 1 2024-04-07T14:31:35.238 INFO:journalctl@ceph.mon.a.smithi083.stdout: index_type: 0 2024-04-07T14:31:35.238 INFO:journalctl@ceph.mon.a.smithi083.stdout: data_block_index_type: 0 2024-04-07T14:31:35.238 INFO:journalctl@ceph.mon.a.smithi083.stdout: index_shortening: 1 2024-04-07T14:31:35.238 INFO:journalctl@ceph.mon.a.smithi083.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-07T14:31:35.238 INFO:journalctl@ceph.mon.a.smithi083.stdout: hash_index_allow_collision: 1 2024-04-07T14:31:35.238 INFO:journalctl@ceph.mon.a.smithi083.stdout: checksum: 1 2024-04-07T14:31:35.238 INFO:journalctl@ceph.mon.a.smithi083.stdout: no_block_cache: 0 2024-04-07T14:31:35.238 INFO:journalctl@ceph.mon.a.smithi083.stdout: block_cache: 0x5641ac3c0170 2024-04-07T14:31:35.238 INFO:journalctl@ceph.mon.a.smithi083.stdout: block_cache_name: BinnedLRUCache 2024-04-07T14:31:35.238 INFO:journalctl@ceph.mon.a.smithi083.stdout: block_cache_options: 2024-04-07T14:31:35.238 INFO:journalctl@ceph.mon.a.smithi083.stdout: capacity : 536870912 2024-04-07T14:31:35.239 INFO:journalctl@ceph.mon.a.smithi083.stdout: num_shard_bits : 4 2024-04-07T14:31:35.239 INFO:journalctl@ceph.mon.a.smithi083.stdout: strict_capacity_limit : 0 2024-04-07T14:31:35.239 INFO:journalctl@ceph.mon.a.smithi083.stdout: high_pri_pool_ratio: 0.000 2024-04-07T14:31:35.239 INFO:journalctl@ceph.mon.a.smithi083.stdout: block_cache_compressed: (nil) 2024-04-07T14:31:35.239 INFO:journalctl@ceph.mon.a.smithi083.stdout: persistent_cache: (nil) 2024-04-07T14:31:35.239 INFO:journalctl@ceph.mon.a.smithi083.stdout: block_size: 4096 2024-04-07T14:31:35.239 INFO:journalctl@ceph.mon.a.smithi083.stdout: block_size_deviation: 10 2024-04-07T14:31:35.239 INFO:journalctl@ceph.mon.a.smithi083.stdout: block_restart_interval: 16 2024-04-07T14:31:35.239 INFO:journalctl@ceph.mon.a.smithi083.stdout: index_block_restart_interval: 1 2024-04-07T14:31:35.239 INFO:journalctl@ceph.mon.a.smithi083.stdout: metadata_block_size: 4096 2024-04-07T14:31:35.239 INFO:journalctl@ceph.mon.a.smithi083.stdout: partition_filters: 0 2024-04-07T14:31:35.239 INFO:journalctl@ceph.mon.a.smithi083.stdout: use_delta_encoding: 1 2024-04-07T14:31:35.239 INFO:journalctl@ceph.mon.a.smithi083.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-07T14:31:35.239 INFO:journalctl@ceph.mon.a.smithi083.stdout: whole_key_filtering: 1 2024-04-07T14:31:35.239 INFO:journalctl@ceph.mon.a.smithi083.stdout: verify_compression: 0 2024-04-07T14:31:35.240 INFO:journalctl@ceph.mon.a.smithi083.stdout: read_amp_bytes_per_bit: 0 2024-04-07T14:31:35.240 INFO:journalctl@ceph.mon.a.smithi083.stdout: format_version: 4 2024-04-07T14:31:35.240 INFO:journalctl@ceph.mon.a.smithi083.stdout: enable_index_compression: 1 2024-04-07T14:31:35.240 INFO:journalctl@ceph.mon.a.smithi083.stdout: block_align: 0 2024-04-07T14:31:35.240 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.write_buffer_size: 33554432 2024-04-07T14:31:35.240 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.max_write_buffer_number: 2 2024-04-07T14:31:35.240 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.compression: NoCompression 2024-04-07T14:31:35.240 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.bottommost_compression: Disabled 2024-04-07T14:31:35.240 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.prefix_extractor: nullptr 2024-04-07T14:31:35.240 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-07T14:31:35.240 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.num_levels: 7 2024-04-07T14:31:35.240 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-07T14:31:35.240 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-07T14:31:35.240 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-07T14:31:35.240 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-07T14:31:35.241 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-07T14:31:35.241 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-07T14:31:35.241 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-07T14:31:35.241 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-07T14:31:35.241 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-07T14:31:35.241 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-07T14:31:35.241 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-07T14:31:35.241 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.compression_opts.level: 32767 2024-04-07T14:31:35.241 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.compression_opts.strategy: 0 2024-04-07T14:31:35.241 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-07T14:31:35.241 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-07T14:31:35.241 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-07T14:31:35.241 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.compression_opts.enabled: false 2024-04-07T14:31:35.241 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-07T14:31:35.241 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-07T14:31:35.242 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-07T14:31:35.242 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.target_file_size_base: 67108864 2024-04-07T14:31:35.242 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-07T14:31:35.242 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-07T14:31:35.242 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-07T14:31:35.242 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-07T14:31:35.242 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-07T14:31:35.242 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-07T14:31:35.242 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-07T14:31:35.242 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-07T14:31:35.242 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-07T14:31:35.242 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-07T14:31:35.242 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-07T14:31:35.242 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-07T14:31:35.243 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-07T14:31:35.243 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.arena_block_size: 4194304 2024-04-07T14:31:35.243 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-07T14:31:35.243 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-07T14:31:35.243 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-07T14:31:35.243 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.disable_auto_compactions: 0 2024-04-07T14:31:35.243 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-07T14:31:35.243 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-07T14:31:35.243 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-07T14:31:35.243 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-07T14:31:35.243 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-07T14:31:35.243 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-07T14:31:35.243 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-07T14:31:35.243 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-07T14:31:35.244 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-07T14:31:35.244 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-07T14:31:35.244 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.table_properties_collectors: 2024-04-07T14:31:35.244 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.inplace_update_support: 0 2024-04-07T14:31:35.244 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-07T14:31:35.244 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-07T14:31:35.244 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-07T14:31:35.244 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-07T14:31:35.244 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.bloom_locality: 0 2024-04-07T14:31:35.244 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.max_successive_merges: 0 2024-04-07T14:31:35.244 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-07T14:31:35.244 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.paranoid_file_checks: 0 2024-04-07T14:31:35.244 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.force_consistency_checks: 1 2024-04-07T14:31:35.244 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.report_bg_io_stats: 0 2024-04-07T14:31:35.244 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.ttl: 2592000 2024-04-07T14:31:35.244 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-07T14:31:35.245 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.enable_blob_files: false 2024-04-07T14:31:35.245 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.min_blob_size: 0 2024-04-07T14:31:35.245 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.blob_file_size: 268435456 2024-04-07T14:31:35.245 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-07T14:31:35.245 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-07T14:31:35.245 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-07T14:31:35.245 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-07T14:31:35.245 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-04-07T14:31:35.245 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-04-07T14:31:35.245 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712500295175082, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-04-07T14:31:35.246 INFO:teuthology.orchestra.run.smithi083.stderr:Setting mon public_network to 172.21.0.0/20 2024-04-07T14:31:35.246 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-04-07T14:31:35.246 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-07T14:31:35.246 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712500295175782, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79179, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77442, "index_size": 225, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10027, "raw_average_key_size": 47, "raw_value_size": 71751, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 211, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1712500295, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "007d43f4-b910-49fd-a3d9-460ad707eadf", "db_session_id": "RU8PJFBVQS1QLXO4979J"}} 2024-04-07T14:31:35.247 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-04-07T14:31:35.247 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712500295176093, "job": 1, "event": "recovery_finished"} 2024-04-07T14:31:35.247 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-07T14:31:35.247 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x5641ac3a6a80 2024-04-07T14:31:35.247 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: DB pointer 0x5641ac3b6000 2024-04-07T14:31:35.247 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-04-07T14:31:35.247 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-07T14:31:35.247 INFO:journalctl@ceph.mon.a.smithi083.stdout: ** DB Stats ** 2024-04-07T14:31:35.247 INFO:journalctl@ceph.mon.a.smithi083.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-07T14:31:35.247 INFO:journalctl@ceph.mon.a.smithi083.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-07T14:31:35.247 INFO:journalctl@ceph.mon.a.smithi083.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-07T14:31:35.247 INFO:journalctl@ceph.mon.a.smithi083.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-07T14:31:35.247 INFO:journalctl@ceph.mon.a.smithi083.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-07T14:31:35.248 INFO:journalctl@ceph.mon.a.smithi083.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-07T14:31:35.248 INFO:journalctl@ceph.mon.a.smithi083.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-07T14:31:35.248 INFO:journalctl@ceph.mon.a.smithi083.stdout: 2024-04-07T14:31:35.248 INFO:journalctl@ceph.mon.a.smithi083.stdout: ** Compaction Stats [default] ** 2024-04-07T14:31:35.248 INFO:journalctl@ceph.mon.a.smithi083.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-07T14:31:35.248 INFO:journalctl@ceph.mon.a.smithi083.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-07T14:31:35.248 INFO:journalctl@ceph.mon.a.smithi083.stdout: L0 2/0 79.08 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 160.3 0.00 0.00 1 0.000 0 0 2024-04-07T14:31:35.248 INFO:journalctl@ceph.mon.a.smithi083.stdout: Sum 2/0 79.08 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 160.3 0.00 0.00 1 0.000 0 0 2024-04-07T14:31:35.248 INFO:journalctl@ceph.mon.a.smithi083.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 160.3 0.00 0.00 1 0.000 0 0 2024-04-07T14:31:35.248 INFO:journalctl@ceph.mon.a.smithi083.stdout: 2024-04-07T14:31:35.248 INFO:journalctl@ceph.mon.a.smithi083.stdout: ** Compaction Stats [default] ** 2024-04-07T14:31:35.248 INFO:journalctl@ceph.mon.a.smithi083.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-07T14:31:35.248 INFO:journalctl@ceph.mon.a.smithi083.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-07T14:31:35.248 INFO:journalctl@ceph.mon.a.smithi083.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 160.3 0.00 0.00 1 0.000 0 0 2024-04-07T14:31:35.249 INFO:journalctl@ceph.mon.a.smithi083.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-07T14:31:35.249 INFO:journalctl@ceph.mon.a.smithi083.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-07T14:31:35.249 INFO:journalctl@ceph.mon.a.smithi083.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-07T14:31:35.249 INFO:journalctl@ceph.mon.a.smithi083.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-07T14:31:35.249 INFO:journalctl@ceph.mon.a.smithi083.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-07T14:31:35.249 INFO:journalctl@ceph.mon.a.smithi083.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-07T14:31:35.249 INFO:journalctl@ceph.mon.a.smithi083.stdout: Cumulative compaction: 0.00 GB write, 25.82 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-07T14:31:35.249 INFO:journalctl@ceph.mon.a.smithi083.stdout: Interval compaction: 0.00 GB write, 25.82 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-07T14:31:35.249 INFO:journalctl@ceph.mon.a.smithi083.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-07T14:31:35.249 INFO:journalctl@ceph.mon.a.smithi083.stdout: 2024-04-07T14:31:35.249 INFO:journalctl@ceph.mon.a.smithi083.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-07T14:31:35.249 INFO:journalctl@ceph.mon.a.smithi083.stdout: 2024-04-07T14:31:35.249 INFO:journalctl@ceph.mon.a.smithi083.stdout: ** Compaction Stats [default] ** 2024-04-07T14:31:35.249 INFO:journalctl@ceph.mon.a.smithi083.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-07T14:31:35.250 INFO:journalctl@ceph.mon.a.smithi083.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-07T14:31:35.250 INFO:journalctl@ceph.mon.a.smithi083.stdout: L0 2/0 79.08 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 160.3 0.00 0.00 1 0.000 0 0 2024-04-07T14:31:35.250 INFO:journalctl@ceph.mon.a.smithi083.stdout: Sum 2/0 79.08 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 160.3 0.00 0.00 1 0.000 0 0 2024-04-07T14:31:35.250 INFO:journalctl@ceph.mon.a.smithi083.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-07T14:31:35.250 INFO:journalctl@ceph.mon.a.smithi083.stdout: 2024-04-07T14:31:35.250 INFO:journalctl@ceph.mon.a.smithi083.stdout: ** Compaction Stats [default] ** 2024-04-07T14:31:35.250 INFO:journalctl@ceph.mon.a.smithi083.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-07T14:31:35.250 INFO:journalctl@ceph.mon.a.smithi083.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-07T14:31:35.250 INFO:journalctl@ceph.mon.a.smithi083.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 160.3 0.00 0.00 1 0.000 0 0 2024-04-07T14:31:35.250 INFO:journalctl@ceph.mon.a.smithi083.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-07T14:31:35.250 INFO:journalctl@ceph.mon.a.smithi083.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-07T14:31:35.250 INFO:journalctl@ceph.mon.a.smithi083.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-07T14:31:35.251 INFO:journalctl@ceph.mon.a.smithi083.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-07T14:31:35.251 INFO:journalctl@ceph.mon.a.smithi083.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-07T14:31:35.251 INFO:journalctl@ceph.mon.a.smithi083.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-07T14:31:35.251 INFO:journalctl@ceph.mon.a.smithi083.stdout: Cumulative compaction: 0.00 GB write, 25.53 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-07T14:31:35.251 INFO:journalctl@ceph.mon.a.smithi083.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-07T14:31:35.251 INFO:journalctl@ceph.mon.a.smithi083.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-07T14:31:35.251 INFO:journalctl@ceph.mon.a.smithi083.stdout: 2024-04-07T14:31:35.251 INFO:journalctl@ceph.mon.a.smithi083.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-07T14:31:35.251 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: starting mon.a rank 0 at public addrs [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] at bind addrs [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 2024-04-07T14:31:35.251 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: mon.a@-1(???) e1 preinit fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 2024-04-07T14:31:35.251 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: mon.a@-1(???).mds e1 new map 2024-04-07T14:31:35.251 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: mon.a@-1(???).mds e1 print_map 2024-04-07T14:31:35.251 INFO:journalctl@ceph.mon.a.smithi083.stdout: e1 2024-04-07T14:31:35.251 INFO:journalctl@ceph.mon.a.smithi083.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-07T14:31:35.251 INFO:journalctl@ceph.mon.a.smithi083.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-07T14:31:35.252 INFO:journalctl@ceph.mon.a.smithi083.stdout: legacy client fscid: -1 2024-04-07T14:31:35.252 INFO:journalctl@ceph.mon.a.smithi083.stdout: 2024-04-07T14:31:35.252 INFO:journalctl@ceph.mon.a.smithi083.stdout: No filesystems configured 2024-04-07T14:31:35.252 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-07T14:31:35.252 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-07T14:31:35.252 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-07T14:31:35.252 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-07T14:31:35.252 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-04-07T14:31:35.252 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: expand_channel_meta expand map: {default=false} 2024-04-07T14:31:35.252 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: expand_channel_meta from 'false' to 'false' 2024-04-07T14:31:35.252 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: expand_channel_meta expanded map: {default=false} 2024-04-07T14:31:35.252 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: expand_channel_meta expand map: {default=info} 2024-04-07T14:31:35.252 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: expand_channel_meta from 'info' to 'info' 2024-04-07T14:31:35.252 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: expand_channel_meta expanded map: {default=info} 2024-04-07T14:31:35.252 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: expand_channel_meta expand map: {default=daemon} 2024-04-07T14:31:35.252 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: expand_channel_meta from 'daemon' to 'daemon' 2024-04-07T14:31:35.253 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: expand_channel_meta expanded map: {default=daemon} 2024-04-07T14:31:35.253 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: expand_channel_meta expand map: {default=debug} 2024-04-07T14:31:35.253 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: expand_channel_meta from 'debug' to 'debug' 2024-04-07T14:31:35.253 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: expand_channel_meta expanded map: {default=debug} 2024-04-07T14:31:35.253 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 bash[24790]: 045b78f11b8e0a839a0de89ffde1b01200a6e2b3f8070084bb844c2a9e030bc8 2024-04-07T14:31:35.253 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 systemd[1]: Started Ceph mon.a for 7388281a-f4eb-11ee-b649-cb9ed24678a4. 2024-04-07T14:31:35.486 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-07T14:31:35.486 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: monmap e1: 1 mons at {a=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0]} 2024-04-07T14:31:35.486 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: fsmap 2024-04-07T14:31:35.486 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: osdmap e1: 0 total, 0 up, 0 in 2024-04-07T14:31:35.486 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:35 smithi083 ceph-mon[24810]: mgrmap e1: no daemons active 2024-04-07T14:31:36.237 INFO:teuthology.orchestra.run.smithi083.stderr:Wrote config to /etc/ceph/ceph.conf 2024-04-07T14:31:36.238 INFO:teuthology.orchestra.run.smithi083.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-04-07T14:31:36.238 INFO:teuthology.orchestra.run.smithi083.stderr:Creating mgr... 2024-04-07T14:31:36.238 INFO:teuthology.orchestra.run.smithi083.stderr:Verifying port 9283 ... 2024-04-07T14:31:36.403 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Failed to reset failed state of unit ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@mgr.y.service: Unit ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@mgr.y.service not loaded. 2024-04-07T14:31:36.411 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: Created symlink /etc/systemd/system/ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4.target.wants/ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@mgr.y.service → /etc/systemd/system/ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@.service. 2024-04-07T14:31:36.676 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:36 smithi083 systemd[1]: Starting Ceph mgr.y for 7388281a-f4eb-11ee-b649-cb9ed24678a4... 2024-04-07T14:31:37.063 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:36 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/3413283473' entity='client.admin' 2024-04-07T14:31:37.063 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:36 smithi083 podman[25030]: 2024-04-07 14:31:36.762015068 +0000 UTC m=+0.098322173 container create 966a9d7367c02082435a2fe3f0164915fcc0829a390f6cde0cc8118c38bae02b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, maintainer=Guillaume Abrioux , release=754, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, io.openshift.expose-services=, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container) 2024-04-07T14:31:37.063 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:36 smithi083 podman[25030]: 2024-04-07 14:31:36.681812207 +0000 UTC m=+0.018119310 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-04-07T14:31:37.063 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:36 smithi083 podman[25030]: 2024-04-07 14:31:36.964280964 +0000 UTC m=+0.300588067 container init 966a9d7367c02082435a2fe3f0164915fcc0829a390f6cde0cc8118c38bae02b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, version=8, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, vendor=Red Hat, Inc., release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-04-07T14:31:37.063 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:36 smithi083 podman[25030]: 2024-04-07 14:31:36.966761215 +0000 UTC m=+0.303068320 container start 966a9d7367c02082435a2fe3f0164915fcc0829a390f6cde0cc8118c38bae02b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, vcs-type=git, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, io.openshift.expose-services=, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux ) 2024-04-07T14:31:37.064 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:37 smithi083 bash[25030]: 966a9d7367c02082435a2fe3f0164915fcc0829a390f6cde0cc8118c38bae02b 2024-04-07T14:31:37.076 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: disabled 2024-04-07T14:31:37.083 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: inactive 2024-04-07T14:31:37.084 INFO:teuthology.orchestra.run.smithi083.stderr:firewalld.service is not enabled 2024-04-07T14:31:37.084 INFO:teuthology.orchestra.run.smithi083.stderr:Not possible to enable service . firewalld.service is not available 2024-04-07T14:31:37.093 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: disabled 2024-04-07T14:31:37.100 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: inactive 2024-04-07T14:31:37.101 INFO:teuthology.orchestra.run.smithi083.stderr:firewalld.service is not enabled 2024-04-07T14:31:37.101 INFO:teuthology.orchestra.run.smithi083.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-04-07T14:31:37.102 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for mgr to start... 2024-04-07T14:31:37.102 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for mgr... 2024-04-07T14:31:37.323 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:37 smithi083 systemd[1]: Started Ceph mgr.y for 7388281a-f4eb-11ee-b649-cb9ed24678a4. 2024-04-07T14:31:37.583 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:37 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:37.466+0000 7fcc00488000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-07T14:31:37.583 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:37 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:37.558+0000 7fcc00488000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-07T14:31:37.586 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-04-07T14:31:37.587 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: { 2024-04-07T14:31:37.587 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "fsid": "7388281a-f4eb-11ee-b649-cb9ed24678a4", 2024-04-07T14:31:37.587 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "health": { 2024-04-07T14:31:37.587 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-07T14:31:37.587 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "checks": {}, 2024-04-07T14:31:37.587 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mutes": [] 2024-04-07T14:31:37.587 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-07T14:31:37.587 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-07T14:31:37.587 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum": [ 2024-04-07T14:31:37.587 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 0 2024-04-07T14:31:37.588 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2024-04-07T14:31:37.588 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-07T14:31:37.588 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "a" 2024-04-07T14:31:37.588 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2024-04-07T14:31:37.588 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-04-07T14:31:37.588 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "monmap": { 2024-04-07T14:31:37.588 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-07T14:31:37.588 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-07T14:31:37.588 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-07T14:31:37.588 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-07T14:31:37.588 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osdmap": { 2024-04-07T14:31:37.588 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-07T14:31:37.588 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-07T14:31:37.589 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-07T14:31:37.589 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-07T14:31:37.589 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-07T14:31:37.592 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-07T14:31:37.593 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-07T14:31:37.593 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-07T14:31:37.593 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "pgmap": { 2024-04-07T14:31:37.593 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-07T14:31:37.593 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-07T14:31:37.593 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-07T14:31:37.593 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-07T14:31:37.593 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-07T14:31:37.593 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-07T14:31:37.593 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-07T14:31:37.593 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-07T14:31:37.593 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-07T14:31:37.593 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "fsmap": { 2024-04-07T14:31:37.593 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-07T14:31:37.593 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-07T14:31:37.593 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-07T14:31:37.594 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-07T14:31:37.594 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-07T14:31:37.594 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "available": false, 2024-04-07T14:31:37.594 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-07T14:31:37.594 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "modules": [ 2024-04-07T14:31:37.594 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "iostat", 2024-04-07T14:31:37.594 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "nfs", 2024-04-07T14:31:37.594 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "restful" 2024-04-07T14:31:37.594 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2024-04-07T14:31:37.594 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "services": {} 2024-04-07T14:31:37.594 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-07T14:31:37.594 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "servicemap": { 2024-04-07T14:31:37.594 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-07T14:31:37.595 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "modified": "2024-04-07T14:31:30.810237+0000", 2024-04-07T14:31:37.595 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "services": {} 2024-04-07T14:31:37.595 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-07T14:31:37.595 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-07T14:31:37.595 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: } 2024-04-07T14:31:37.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:37 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/1494155114' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-07T14:31:38.197 INFO:teuthology.orchestra.run.smithi083.stderr:mgr not available, waiting (1/15)... 2024-04-07T14:31:38.426 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:38 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:38.161+0000 7fcc00488000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-07T14:31:38.426 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:38 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:38.249+0000 7fcc00488000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-07T14:31:38.426 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:38 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:38.331+0000 7fcc00488000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-07T14:31:38.742 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:38 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:38.477+0000 7fcc00488000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-07T14:31:38.742 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:38 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:38.555+0000 7fcc00488000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-07T14:31:38.742 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:38 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:38.661+0000 7fcc00488000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-07T14:31:38.742 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:38 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:38.741+0000 7fcc00488000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-07T14:31:39.176 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:38 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:38.901+0000 7fcc00488000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-07T14:31:39.176 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:38 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:38.984+0000 7fcc00488000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-07T14:31:40.321 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:40 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:39.999+0000 7fcc00488000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-07T14:31:40.321 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:40 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:40.126+0000 7fcc00488000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-07T14:31:40.321 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:40 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:40.321+0000 7fcc00488000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-07T14:31:40.647 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:40 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:40.407+0000 7fcc00488000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-07T14:31:40.647 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:40 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:40.544+0000 7fcc00488000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-07T14:31:40.727 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-04-07T14:31:40.727 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: { 2024-04-07T14:31:40.727 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "fsid": "7388281a-f4eb-11ee-b649-cb9ed24678a4", 2024-04-07T14:31:40.727 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "health": { 2024-04-07T14:31:40.727 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-07T14:31:40.727 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "checks": {}, 2024-04-07T14:31:40.727 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mutes": [] 2024-04-07T14:31:40.727 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-07T14:31:40.727 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-07T14:31:40.728 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum": [ 2024-04-07T14:31:40.728 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 0 2024-04-07T14:31:40.728 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2024-04-07T14:31:40.728 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-07T14:31:40.728 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "a" 2024-04-07T14:31:40.728 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2024-04-07T14:31:40.728 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum_age": 5, 2024-04-07T14:31:40.728 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "monmap": { 2024-04-07T14:31:40.729 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-07T14:31:40.729 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-07T14:31:40.729 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-07T14:31:40.729 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-07T14:31:40.729 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osdmap": { 2024-04-07T14:31:40.729 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-07T14:31:40.730 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-07T14:31:40.730 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-07T14:31:40.730 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-07T14:31:40.730 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-07T14:31:40.733 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-07T14:31:40.733 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-07T14:31:40.733 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-07T14:31:40.733 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "pgmap": { 2024-04-07T14:31:40.734 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-07T14:31:40.734 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-07T14:31:40.734 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-07T14:31:40.734 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-07T14:31:40.734 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-07T14:31:40.734 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-07T14:31:40.734 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-07T14:31:40.734 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-07T14:31:40.734 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-07T14:31:40.734 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "fsmap": { 2024-04-07T14:31:40.734 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-07T14:31:40.734 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-07T14:31:40.734 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-07T14:31:40.735 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-07T14:31:40.735 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-07T14:31:40.735 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "available": false, 2024-04-07T14:31:40.735 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-07T14:31:40.735 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "modules": [ 2024-04-07T14:31:40.735 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "iostat", 2024-04-07T14:31:40.735 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "nfs", 2024-04-07T14:31:40.735 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "restful" 2024-04-07T14:31:40.735 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2024-04-07T14:31:40.735 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "services": {} 2024-04-07T14:31:40.735 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-07T14:31:40.735 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "servicemap": { 2024-04-07T14:31:40.735 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-07T14:31:40.735 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "modified": "2024-04-07T14:31:30.810237+0000", 2024-04-07T14:31:40.735 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "services": {} 2024-04-07T14:31:40.736 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-07T14:31:40.736 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-07T14:31:40.736 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: } 2024-04-07T14:31:40.926 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:40 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:40.870+0000 7fcc00488000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-07T14:31:41.343 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:41 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/3604976316' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-07T14:31:41.344 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:41 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:41.090+0000 7fcc00488000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-07T14:31:41.344 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:41 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:41.173+0000 7fcc00488000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-07T14:31:41.344 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:41 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:41.343+0000 7fcc00488000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-07T14:31:41.370 INFO:teuthology.orchestra.run.smithi083.stderr:mgr not available, waiting (2/15)... 2024-04-07T14:31:42.543 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:42 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:42.260+0000 7fcc00488000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-07T14:31:42.544 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:42 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:42.543+0000 7fcc00488000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-07T14:31:42.845 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:42 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:42.636+0000 7fcc00488000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-07T14:31:42.845 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:42 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:42.780+0000 7fcc00488000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-07T14:31:43.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:42 smithi083 ceph-mon[24810]: Activating manager daemon y 2024-04-07T14:31:43.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:42 smithi083 ceph-mon[24810]: mgrmap e2: y(active, starting, since 0.00289188s) 2024-04-07T14:31:43.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:42 smithi083 ceph-mon[24810]: from='mgr.14100 172.21.15.83:0/1034146771' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-07T14:31:43.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:42 smithi083 ceph-mon[24810]: from='mgr.14100 172.21.15.83:0/1034146771' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-07T14:31:43.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:42 smithi083 ceph-mon[24810]: from='mgr.14100 172.21.15.83:0/1034146771' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-07T14:31:43.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:42 smithi083 ceph-mon[24810]: from='mgr.14100 172.21.15.83:0/1034146771' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-07T14:31:43.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:42 smithi083 ceph-mon[24810]: from='mgr.14100 172.21.15.83:0/1034146771' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-07T14:31:43.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:42 smithi083 ceph-mon[24810]: Manager daemon y is now available 2024-04-07T14:31:43.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:42 smithi083 ceph-mon[24810]: from='mgr.14100 172.21.15.83:0/1034146771' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:31:43.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:42 smithi083 ceph-mon[24810]: from='mgr.14100 172.21.15.83:0/1034146771' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:31:43.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:42 smithi083 ceph-mon[24810]: from='mgr.14100 172.21.15.83:0/1034146771' entity='mgr.y' 2024-04-07T14:31:43.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:42 smithi083 ceph-mon[24810]: from='mgr.14100 172.21.15.83:0/1034146771' entity='mgr.y' 2024-04-07T14:31:43.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:42 smithi083 ceph-mon[24810]: from='mgr.14100 172.21.15.83:0/1034146771' entity='mgr.y' 2024-04-07T14:31:43.824 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-04-07T14:31:43.825 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: { 2024-04-07T14:31:43.825 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "fsid": "7388281a-f4eb-11ee-b649-cb9ed24678a4", 2024-04-07T14:31:43.825 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "health": { 2024-04-07T14:31:43.825 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-07T14:31:43.825 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "checks": {}, 2024-04-07T14:31:43.825 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mutes": [] 2024-04-07T14:31:43.825 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-07T14:31:43.825 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-07T14:31:43.825 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum": [ 2024-04-07T14:31:43.825 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 0 2024-04-07T14:31:43.825 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2024-04-07T14:31:43.825 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-07T14:31:43.825 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "a" 2024-04-07T14:31:43.825 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2024-04-07T14:31:43.825 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "quorum_age": 8, 2024-04-07T14:31:43.826 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "monmap": { 2024-04-07T14:31:43.826 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-07T14:31:43.826 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-07T14:31:43.826 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-07T14:31:43.826 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-07T14:31:43.826 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osdmap": { 2024-04-07T14:31:43.826 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-07T14:31:43.826 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-07T14:31:43.826 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-07T14:31:43.826 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-07T14:31:43.826 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-07T14:31:43.830 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-07T14:31:43.830 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-07T14:31:43.830 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-07T14:31:43.830 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "pgmap": { 2024-04-07T14:31:43.830 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-07T14:31:43.830 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-07T14:31:43.830 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-07T14:31:43.830 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-07T14:31:43.830 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-07T14:31:43.830 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-07T14:31:43.830 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-07T14:31:43.830 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-07T14:31:43.831 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-07T14:31:43.831 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "fsmap": { 2024-04-07T14:31:43.831 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-07T14:31:43.831 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-07T14:31:43.831 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-07T14:31:43.831 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-07T14:31:43.831 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-07T14:31:43.831 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "available": true, 2024-04-07T14:31:43.831 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-07T14:31:43.831 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "modules": [ 2024-04-07T14:31:43.831 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "iostat", 2024-04-07T14:31:43.831 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "nfs", 2024-04-07T14:31:43.831 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "restful" 2024-04-07T14:31:43.831 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ], 2024-04-07T14:31:43.831 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "services": {} 2024-04-07T14:31:43.832 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-07T14:31:43.832 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "servicemap": { 2024-04-07T14:31:43.832 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-07T14:31:43.832 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "modified": "2024-04-07T14:31:30.810237+0000", 2024-04-07T14:31:43.832 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "services": {} 2024-04-07T14:31:43.832 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: }, 2024-04-07T14:31:43.832 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-07T14:31:43.832 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: } 2024-04-07T14:31:44.372 INFO:teuthology.orchestra.run.smithi083.stderr:mgr is available 2024-04-07T14:31:44.923 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:44 smithi083 ceph-mon[24810]: mgrmap e3: y(active, since 1.00571s) 2024-04-07T14:31:44.923 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:44 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/2246662903' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-07T14:31:44.925 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-04-07T14:31:44.925 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: [global] 2024-04-07T14:31:44.925 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: fsid = 7388281a-f4eb-11ee-b649-cb9ed24678a4 2024-04-07T14:31:44.925 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-04-07T14:31:44.925 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-04-07T14:31:44.925 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-04-07T14:31:44.925 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-04-07T14:31:44.925 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-04-07T14:31:44.925 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: [mgr] 2024-04-07T14:31:44.925 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-04-07T14:31:44.925 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 2024-04-07T14:31:44.926 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: [osd] 2024-04-07T14:31:44.926 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-04-07T14:31:44.926 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-04-07T14:31:45.462 INFO:teuthology.orchestra.run.smithi083.stderr:Enabling cephadm module... 2024-04-07T14:31:46.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:45 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/1062466845' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-07T14:31:46.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:45 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/1062466845' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-07T14:31:47.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:46 smithi083 ceph-mon[24810]: mgrmap e4: y(active, since 3s) 2024-04-07T14:31:47.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:46 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/2750352740' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-04-07T14:31:47.176 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:46 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ignoring --setuser ceph since I am not root 2024-04-07T14:31:47.176 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:46 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ignoring --setgroup ceph since I am not root 2024-04-07T14:31:47.676 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:47 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:47.301+0000 7fe94aff5000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-07T14:31:47.676 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:47 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:47.390+0000 7fe94aff5000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-07T14:31:48.074 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:47 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/2750352740' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-04-07T14:31:48.075 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:47 smithi083 ceph-mon[24810]: mgrmap e5: y(active, since 4s) 2024-04-07T14:31:48.075 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:47 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:47.986+0000 7fe94aff5000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-07T14:31:48.383 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:48 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:48.074+0000 7fe94aff5000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-07T14:31:48.384 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:48 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:48.157+0000 7fe94aff5000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-07T14:31:48.384 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:48 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:48.303+0000 7fe94aff5000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-07T14:31:48.384 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:48 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:48.383+0000 7fe94aff5000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-07T14:31:48.676 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:48 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:48.491+0000 7fe94aff5000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-07T14:31:48.676 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:48 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:48.575+0000 7fe94aff5000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-07T14:31:49.176 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:48 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:48.736+0000 7fe94aff5000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-07T14:31:49.176 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:48 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:48.822+0000 7fe94aff5000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-07T14:31:50.154 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:49 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:49.840+0000 7fe94aff5000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-07T14:31:50.155 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:49 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:49.967+0000 7fe94aff5000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-07T14:31:50.155 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:50 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:50.154+0000 7fe94aff5000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-07T14:31:50.413 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:50 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:50.241+0000 7fe94aff5000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-07T14:31:50.413 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:50 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:50.373+0000 7fe94aff5000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-07T14:31:50.993 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:50 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:50.686+0000 7fe94aff5000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-07T14:31:50.994 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:50 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:50.908+0000 7fe94aff5000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-07T14:31:50.994 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:50 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:50.993+0000 7fe94aff5000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-07T14:31:51.426 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:51 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:51.162+0000 7fe94aff5000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-07T14:31:52.357 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:52 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:52.068+0000 7fe94aff5000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-07T14:31:52.676 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:52 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:52.357+0000 7fe94aff5000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-07T14:31:52.676 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:52 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:52.451+0000 7fe94aff5000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-07T14:31:52.676 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:52 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:31:52.598+0000 7fe94aff5000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-07T14:31:53.907 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:53 smithi083 ceph-mon[24810]: Active manager daemon y restarted 2024-04-07T14:31:53.907 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:53 smithi083 ceph-mon[24810]: Activating manager daemon y 2024-04-07T14:31:53.907 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:53 smithi083 ceph-mon[24810]: osdmap e2: 0 total, 0 up, 0 in 2024-04-07T14:31:53.907 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:53 smithi083 ceph-mon[24810]: mgrmap e6: y(active, starting, since 0.0524284s) 2024-04-07T14:31:53.907 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:53 smithi083 ceph-mon[24810]: from='mgr.14118 172.21.15.83:0/3767842742' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-07T14:31:53.907 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:53 smithi083 ceph-mon[24810]: from='mgr.14118 172.21.15.83:0/3767842742' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-07T14:31:53.908 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:53 smithi083 ceph-mon[24810]: from='mgr.14118 172.21.15.83:0/3767842742' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-07T14:31:53.908 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:53 smithi083 ceph-mon[24810]: from='mgr.14118 172.21.15.83:0/3767842742' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-07T14:31:53.908 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:53 smithi083 ceph-mon[24810]: from='mgr.14118 172.21.15.83:0/3767842742' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-07T14:31:53.908 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:53 smithi083 ceph-mon[24810]: Manager daemon y is now available 2024-04-07T14:31:53.908 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:53 smithi083 ceph-mon[24810]: from='mgr.14118 172.21.15.83:0/3767842742' entity='mgr.y' 2024-04-07T14:31:53.908 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:53 smithi083 ceph-mon[24810]: from='mgr.14118 172.21.15.83:0/3767842742' entity='mgr.y' 2024-04-07T14:31:53.908 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:53 smithi083 ceph-mon[24810]: from='mgr.14118 172.21.15.83:0/3767842742' entity='mgr.y' 2024-04-07T14:31:53.908 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:53 smithi083 ceph-mon[24810]: from='mgr.14118 172.21.15.83:0/3767842742' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:31:53.908 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:53 smithi083 ceph-mon[24810]: from='mgr.14118 172.21.15.83:0/3767842742' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:31:53.908 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:53 smithi083 ceph-mon[24810]: from='mgr.14118 172.21.15.83:0/3767842742' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:31:53.908 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:53 smithi083 ceph-mon[24810]: from='mgr.14118 172.21.15.83:0/3767842742' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:31:53.908 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:53 smithi083 ceph-mon[24810]: from='mgr.14118 172.21.15.83:0/3767842742' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:31:54.176 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:53 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: [07/Apr/2024:14:31:53] ENGINE Bus STARTING 2024-04-07T14:31:54.176 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:54 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: [07/Apr/2024:14:31:54] ENGINE Serving on https://172.21.15.83:7150 2024-04-07T14:31:54.176 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:31:54 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: [07/Apr/2024:14:31:54] ENGINE Bus STARTED 2024-04-07T14:31:54.620 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: { 2024-04-07T14:31:54.620 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 7, 2024-04-07T14:31:54.620 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "available": true, 2024-04-07T14:31:54.620 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "active_name": "y", 2024-04-07T14:31:54.620 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-07T14:31:54.621 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: } 2024-04-07T14:31:54.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:54 smithi083 ceph-mon[24810]: mgrmap e7: y(active, since 1.0549s) 2024-04-07T14:31:54.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:54 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/2353989573' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-07T14:31:54.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:54 smithi083 ceph-mon[24810]: from='mgr.14118 172.21.15.83:0/3767842742' entity='mgr.y' 2024-04-07T14:31:54.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:54 smithi083 ceph-mon[24810]: from='mgr.14118 172.21.15.83:0/3767842742' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:31:55.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:55 smithi083 ceph-mon[24810]: [07/Apr/2024:14:31:53] ENGINE Bus STARTING 2024-04-07T14:31:55.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:55 smithi083 ceph-mon[24810]: [07/Apr/2024:14:31:54] ENGINE Serving on https://172.21.15.83:7150 2024-04-07T14:31:55.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:55 smithi083 ceph-mon[24810]: [07/Apr/2024:14:31:54] ENGINE Bus STARTED 2024-04-07T14:31:55.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:31:55 smithi083 ceph-mon[24810]: mgrmap e8: y(active, since 2s) 2024-04-07T14:31:59.863 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for the mgr to restart... 2024-04-07T14:31:59.863 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for mgr epoch 7... 2024-04-07T14:32:00.317 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: { 2024-04-07T14:32:00.317 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mgrmap_epoch": 8, 2024-04-07T14:32:00.317 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "initialized": true 2024-04-07T14:32:00.317 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: } 2024-04-07T14:32:00.929 INFO:teuthology.orchestra.run.smithi083.stderr:mgr epoch 7 is available 2024-04-07T14:32:00.930 INFO:teuthology.orchestra.run.smithi083.stderr:Setting orchestrator backend to cephadm... 2024-04-07T14:32:01.534 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:01 smithi083 ceph-mon[24810]: from='client.14128 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-07T14:32:01.535 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:01 smithi083 ceph-mon[24810]: from='client.14128 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-07T14:32:02.619 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:02 smithi083 ceph-mon[24810]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:32:02.620 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:02 smithi083 ceph-mon[24810]: from='mgr.14118 172.21.15.83:0/3767842742' entity='mgr.y' 2024-04-07T14:32:02.620 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:02 smithi083 ceph-mon[24810]: from='mgr.14118 172.21.15.83:0/3767842742' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:32:02.620 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: value unchanged 2024-04-07T14:32:03.172 INFO:teuthology.orchestra.run.smithi083.stderr:Generating ssh key... 2024-04-07T14:32:03.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:03 smithi083 ceph-mon[24810]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:32:04.239 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: Generating public/private rsa key pair. 2024-04-07T14:32:04.239 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: Your identification has been saved in /tmp/tmp5iod5trd/key. 2024-04-07T14:32:04.239 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: Your public key has been saved in /tmp/tmp5iod5trd/key.pub. 2024-04-07T14:32:04.239 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: The key fingerprint is: 2024-04-07T14:32:04.240 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: SHA256:ooo2BF8bQF1l1jWbAi9AHdO1LlqEd985e+H2/ufjBYQ ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4 2024-04-07T14:32:04.240 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: The key's randomart image is: 2024-04-07T14:32:04.240 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: +---[RSA 3072]----+ 2024-04-07T14:32:04.240 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: | ... o+oB+ o+ | 2024-04-07T14:32:04.240 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: | . . +.=o * | 2024-04-07T14:32:04.240 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: | . o = E . | 2024-04-07T14:32:04.240 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: |. o + + o ..| 2024-04-07T14:32:04.240 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: |.. . o. S o . o+.| 2024-04-07T14:32:04.240 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: | .. .. . o . ..+| 2024-04-07T14:32:04.240 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: |. . . +o| 2024-04-07T14:32:04.240 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: | + . ..=| 2024-04-07T14:32:04.240 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: |o o .+O| 2024-04-07T14:32:04.240 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: +----[SHA256]-----+ 2024-04-07T14:32:04.795 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:04 smithi083 ceph-mon[24810]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:32:04.795 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:04 smithi083 ceph-mon[24810]: Generating ssh key... 2024-04-07T14:32:04.795 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:04 smithi083 ceph-mon[24810]: from='mgr.14118 172.21.15.83:0/3767842742' entity='mgr.y' 2024-04-07T14:32:04.795 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:04 smithi083 ceph-mon[24810]: from='mgr.14118 172.21.15.83:0/3767842742' entity='mgr.y' 2024-04-07T14:32:05.156 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDs2zFtKc7hISMkBZWjtnKJt/Yhz+Ix6drlvV4GL+G7UJqGv84/+fFqEnEMelQluxqPwsbflf2FOHuC3m+sKqlKjnAcjNn4SofvmQRVQlAGON/XtaS/RUO/rU9i+TKdYaO/Eo8PTzXt8zrH4yPIdJRCbHCjSdfhJHeYtfaF/GGD1M0TZMjY6Bkt9ic7Vql1JXMtH46N7TVOnW37IB29i0P5BKoHTCcLg/OivlBqLeQmm/JejtCiGSszNSiwHXNZKt++OygCBVPmKt14kzFSfCCbOvf41f/r08N4IFLsOdjpx7oYyXkZ+bDME/N3M5jEKBdi5cAbpfdS/D5IOBo8jjXBXzyyDJMSq0wetwtj5zF61wD7qeajARDxS3Oa6pfgLL02SG2hqOaZJUO9DzDVMd9LA6W6mH1ecMmylNMouCbcgpXY/ApnSmgLmb1ZO2Ab9t2WRmBxlQxod9ApeYYjG4nPMwH1/xVGoXNmw6bIwjZaPK/xnes9z5NDJOnXeCjaa90= ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4 2024-04-07T14:32:05.869 INFO:teuthology.orchestra.run.smithi083.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-04-07T14:32:05.869 INFO:teuthology.orchestra.run.smithi083.stderr:Adding key to root@localhost authorized_keys... 2024-04-07T14:32:05.869 INFO:teuthology.orchestra.run.smithi083.stderr:Adding host smithi083... 2024-04-07T14:32:06.496 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:06 smithi083 ceph-mon[24810]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:32:07.480 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:07 smithi083 ceph-mon[24810]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi083", "addr": "172.21.15.83", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:32:07.490 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: Added host 'smithi083' with addr '172.21.15.83' 2024-04-07T14:32:08.076 INFO:teuthology.orchestra.run.smithi083.stderr:Deploying unmanaged mon service... 2024-04-07T14:32:08.674 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:08 smithi083 ceph-mon[24810]: Deploying cephadm binary to smithi083 2024-04-07T14:32:08.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:08 smithi083 ceph-mon[24810]: from='mgr.14118 172.21.15.83:0/3767842742' entity='mgr.y' 2024-04-07T14:32:08.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:08 smithi083 ceph-mon[24810]: Added host smithi083 2024-04-07T14:32:08.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:08 smithi083 ceph-mon[24810]: from='mgr.14118 172.21.15.83:0/3767842742' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:32:08.675 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: Scheduled mon update... 2024-04-07T14:32:09.418 INFO:teuthology.orchestra.run.smithi083.stderr:Deploying unmanaged mgr service... 2024-04-07T14:32:09.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:09 smithi083 ceph-mon[24810]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:32:09.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:09 smithi083 ceph-mon[24810]: Saving service mon spec with placement count:5 2024-04-07T14:32:09.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:09 smithi083 ceph-mon[24810]: from='mgr.14118 172.21.15.83:0/3767842742' entity='mgr.y' 2024-04-07T14:32:10.076 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-04-07T14:32:11.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:11 smithi083 ceph-mon[24810]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:32:11.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:11 smithi083 ceph-mon[24810]: Saving service mgr spec with placement count:2 2024-04-07T14:32:11.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:11 smithi083 ceph-mon[24810]: from='mgr.14118 172.21.15.83:0/3767842742' entity='mgr.y' 2024-04-07T14:32:11.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:11 smithi083 ceph-mon[24810]: from='mgr.14118 172.21.15.83:0/3767842742' entity='mgr.y' 2024-04-07T14:32:13.046 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:12 smithi083 ceph-mon[24810]: from='mgr.14118 172.21.15.83:0/3767842742' entity='mgr.y' 2024-04-07T14:32:13.046 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:12 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/2293715177' entity='client.admin' 2024-04-07T14:32:13.923 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:13 smithi083 ceph-mon[24810]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:32:13.923 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:13 smithi083 ceph-mon[24810]: from='mgr.14118 172.21.15.83:0/3767842742' entity='mgr.y' 2024-04-07T14:32:13.923 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:13 smithi083 ceph-mon[24810]: from='mgr.14118 172.21.15.83:0/3767842742' entity='mgr.y' 2024-04-07T14:32:13.923 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:13 smithi083 ceph-mon[24810]: from='mgr.14118 172.21.15.83:0/3767842742' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi083", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:32:13.924 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:13 smithi083 ceph-mon[24810]: from='mgr.14118 172.21.15.83:0/3767842742' entity='mgr.y' 2024-04-07T14:32:13.924 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:13 smithi083 ceph-mon[24810]: from='mgr.14118 172.21.15.83:0/3767842742' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi083", "caps": []}]: dispatch 2024-04-07T14:32:13.924 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:13 smithi083 ceph-mon[24810]: from='mgr.14118 172.21.15.83:0/3767842742' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.agent.smithi083", "caps": []}]': finished 2024-04-07T14:32:14.540 INFO:teuthology.orchestra.run.smithi083.stderr:Enabling the dashboard module... 2024-04-07T14:32:15.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:14 smithi083 ceph-mon[24810]: Deploying daemon agent.smithi083 on smithi083 2024-04-07T14:32:15.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:14 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/3235733922' entity='client.admin' 2024-04-07T14:32:16.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:15 smithi083 ceph-mon[24810]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:32:16.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:15 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/4081809111' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-04-07T14:32:16.502 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:16 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ignoring --setuser ceph since I am not root 2024-04-07T14:32:16.503 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:16 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ignoring --setgroup ceph since I am not root 2024-04-07T14:32:16.776 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:16 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:32:16.687+0000 7faeabb97000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-07T14:32:16.776 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:16 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:32:16.776+0000 7faeabb97000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-07T14:32:17.380 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:17 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/4081809111' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-04-07T14:32:17.381 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:17 smithi083 ceph-mon[24810]: mgrmap e9: y(active, since 23s) 2024-04-07T14:32:17.494 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: { 2024-04-07T14:32:17.495 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "epoch": 9, 2024-04-07T14:32:17.495 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "available": true, 2024-04-07T14:32:17.495 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "active_name": "y", 2024-04-07T14:32:17.495 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-07T14:32:17.495 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: } 2024-04-07T14:32:17.676 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:17 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:32:17.381+0000 7faeabb97000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-07T14:32:17.676 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:17 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:32:17.469+0000 7faeabb97000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-07T14:32:17.676 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:17 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:32:17.555+0000 7faeabb97000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-07T14:32:17.976 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:17 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:32:17.703+0000 7faeabb97000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-07T14:32:17.976 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:17 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:32:17.784+0000 7faeabb97000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-07T14:32:17.976 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:17 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:32:17.893+0000 7faeabb97000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-07T14:32:17.976 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:17 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:32:17.976+0000 7faeabb97000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-07T14:32:18.163 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for the mgr to restart... 2024-04-07T14:32:18.163 INFO:teuthology.orchestra.run.smithi083.stderr:Waiting for mgr epoch 9... 2024-04-07T14:32:18.227 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:18 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/1084588207' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-07T14:32:18.227 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:18 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:32:18.138+0000 7faeabb97000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-07T14:32:18.227 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:18 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:32:18.227+0000 7faeabb97000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-07T14:32:19.569 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:19 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:32:19.251+0000 7faeabb97000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-07T14:32:19.569 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:19 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:32:19.380+0000 7faeabb97000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-07T14:32:19.569 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:19 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:32:19.569+0000 7faeabb97000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-07T14:32:19.926 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:19 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:32:19.655+0000 7faeabb97000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-07T14:32:19.926 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:19 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:32:19.789+0000 7faeabb97000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-07T14:32:20.407 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:20 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:32:20.104+0000 7faeabb97000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-07T14:32:20.408 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:20 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:32:20.324+0000 7faeabb97000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-07T14:32:20.408 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:20 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:32:20.408+0000 7faeabb97000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-07T14:32:20.676 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:20 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:32:20.577+0000 7faeabb97000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-07T14:32:21.779 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:21 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:32:21.486+0000 7faeabb97000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-07T14:32:21.779 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:21 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:32:21.779+0000 7faeabb97000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-07T14:32:22.149 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:21 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:32:21.872+0000 7faeabb97000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-07T14:32:22.149 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:22 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:32:22.021+0000 7faeabb97000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-07T14:32:22.426 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:22 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: [07/Apr/2024:14:32:22] ENGINE Bus STARTING 2024-04-07T14:32:22.834 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:22 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: [07/Apr/2024:14:32:22] ENGINE Serving on https://172.21.15.83:7150 2024-04-07T14:32:22.834 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:22 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: [07/Apr/2024:14:32:22] ENGINE Bus STARTED 2024-04-07T14:32:23.084 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:23 smithi083 ceph-mon[24810]: Active manager daemon y restarted 2024-04-07T14:32:23.084 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:23 smithi083 ceph-mon[24810]: Activating manager daemon y 2024-04-07T14:32:23.084 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:23 smithi083 ceph-mon[24810]: osdmap e3: 0 total, 0 up, 0 in 2024-04-07T14:32:23.084 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:23 smithi083 ceph-mon[24810]: mgrmap e10: y(active, starting, since 0.0550162s) 2024-04-07T14:32:23.084 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:23 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-07T14:32:23.084 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:23 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-07T14:32:23.085 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:23 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-07T14:32:23.085 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:23 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-07T14:32:23.085 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:23 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-07T14:32:23.085 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:23 smithi083 ceph-mon[24810]: Manager daemon y is now available 2024-04-07T14:32:23.085 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:23 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:32:23.085 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:23 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:32:23.085 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:23 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:32:23.085 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:23 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:32:23.085 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:23 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:32:23.085 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:23 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:32:23.098 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: { 2024-04-07T14:32:23.098 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-04-07T14:32:23.098 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: "initialized": true 2024-04-07T14:32:23.098 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: } 2024-04-07T14:32:23.899 INFO:teuthology.orchestra.run.smithi083.stderr:mgr epoch 9 is available 2024-04-07T14:32:23.899 INFO:teuthology.orchestra.run.smithi083.stderr:Generating a dashboard self-signed certificate... 2024-04-07T14:32:24.163 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:24 smithi083 ceph-mon[24810]: [07/Apr/2024:14:32:22] ENGINE Bus STARTING 2024-04-07T14:32:24.163 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:24 smithi083 ceph-mon[24810]: [07/Apr/2024:14:32:22] ENGINE Serving on https://172.21.15.83:7150 2024-04-07T14:32:24.163 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:24 smithi083 ceph-mon[24810]: [07/Apr/2024:14:32:22] ENGINE Bus STARTED 2024-04-07T14:32:24.163 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:24 smithi083 ceph-mon[24810]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-07T14:32:24.163 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:24 smithi083 ceph-mon[24810]: mgrmap e11: y(active, since 1.05779s) 2024-04-07T14:32:24.163 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:24 smithi083 ceph-mon[24810]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-07T14:32:24.163 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:24 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:32:24.163 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:24 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi083", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:32:24.163 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:24 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:32:24.163 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:24 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi083", "caps": []}]: dispatch 2024-04-07T14:32:24.638 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: Self-signed certificate created 2024-04-07T14:32:25.287 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:25 smithi083 ceph-mon[24810]: Deploying daemon agent.smithi083 on smithi083 2024-04-07T14:32:25.287 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:25 smithi083 ceph-mon[24810]: mgrmap e12: y(active, since 2s) 2024-04-07T14:32:25.287 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:25 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:32:25.287 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:25 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:32:25.289 INFO:teuthology.orchestra.run.smithi083.stderr:Creating initial admin user... 2024-04-07T14:32:26.170 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$jpos60Mb25Pp6Ztf3Zonf.XSgqtmMi7XLiCS8CLhJpE3qA1yMv4RG", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1712500346, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-04-07T14:32:26.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:26 smithi083 ceph-mon[24810]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:32:26.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:26 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:32:26.797 INFO:teuthology.orchestra.run.smithi083.stderr:Fetching dashboard port number... 2024-04-07T14:32:27.423 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:27 smithi083 ceph-mon[24810]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:32:27.423 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: 8443 2024-04-07T14:32:28.071 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: disabled 2024-04-07T14:32:28.078 INFO:teuthology.orchestra.run.smithi083.stderr:systemctl: inactive 2024-04-07T14:32:28.079 INFO:teuthology.orchestra.run.smithi083.stderr:firewalld.service is not enabled 2024-04-07T14:32:28.079 INFO:teuthology.orchestra.run.smithi083.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-04-07T14:32:28.081 INFO:teuthology.orchestra.run.smithi083.stderr:Ceph Dashboard is now available at: 2024-04-07T14:32:28.081 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-04-07T14:32:28.081 INFO:teuthology.orchestra.run.smithi083.stderr: URL: https://smithi083.front.sepia.ceph.com:8443/ 2024-04-07T14:32:28.081 INFO:teuthology.orchestra.run.smithi083.stderr: User: admin 2024-04-07T14:32:28.082 INFO:teuthology.orchestra.run.smithi083.stderr: Password: k92a1zu9m4 2024-04-07T14:32:28.082 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-04-07T14:32:28.082 INFO:teuthology.orchestra.run.smithi083.stderr:Enabling autotune for osd_memory_target 2024-04-07T14:32:28.334 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:28 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/1020303787' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-04-07T14:32:29.343 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:29 smithi083 ceph-mon[24810]: mgrmap e13: y(active, since 6s) 2024-04-07T14:32:30.132 INFO:teuthology.orchestra.run.smithi083.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-04-07T14:32:30.761 INFO:teuthology.orchestra.run.smithi083.stderr:You can access the Ceph CLI with: 2024-04-07T14:32:30.761 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-04-07T14:32:30.761 INFO:teuthology.orchestra.run.smithi083.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-04-07T14:32:30.761 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-04-07T14:32:30.761 INFO:teuthology.orchestra.run.smithi083.stderr:Please consider enabling telemetry to help improve Ceph: 2024-04-07T14:32:30.762 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-04-07T14:32:30.762 INFO:teuthology.orchestra.run.smithi083.stderr: ceph telemetry on 2024-04-07T14:32:30.762 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-04-07T14:32:30.762 INFO:teuthology.orchestra.run.smithi083.stderr:For more information see: 2024-04-07T14:32:30.762 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-04-07T14:32:30.762 INFO:teuthology.orchestra.run.smithi083.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-04-07T14:32:30.762 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-04-07T14:32:30.762 INFO:teuthology.orchestra.run.smithi083.stderr:Bootstrap complete. 2024-04-07T14:32:30.947 INFO:tasks.cephadm:Fetching config... 2024-04-07T14:32:30.948 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-04-07T14:32:30.948 DEBUG:teuthology.orchestra.run.smithi083:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-04-07T14:32:30.963 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-04-07T14:32:30.963 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-04-07T14:32:30.963 DEBUG:teuthology.orchestra.run.smithi083:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-04-07T14:32:31.017 INFO:tasks.cephadm:Fetching mon keyring... 2024-04-07T14:32:31.018 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-04-07T14:32:31.018 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd if=/var/lib/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/mon.a/keyring of=/dev/stdout 2024-04-07T14:32:31.130 INFO:tasks.cephadm:Fetching pub ssh key... 2024-04-07T14:32:31.130 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-04-07T14:32:31.130 DEBUG:teuthology.orchestra.run.smithi083:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-04-07T14:32:31.186 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-04-07T14:32:31.186 DEBUG:teuthology.orchestra.run.smithi083:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDs2zFtKc7hISMkBZWjtnKJt/Yhz+Ix6drlvV4GL+G7UJqGv84/+fFqEnEMelQluxqPwsbflf2FOHuC3m+sKqlKjnAcjNn4SofvmQRVQlAGON/XtaS/RUO/rU9i+TKdYaO/Eo8PTzXt8zrH4yPIdJRCbHCjSdfhJHeYtfaF/GGD1M0TZMjY6Bkt9ic7Vql1JXMtH46N7TVOnW37IB29i0P5BKoHTCcLg/OivlBqLeQmm/JejtCiGSszNSiwHXNZKt++OygCBVPmKt14kzFSfCCbOvf41f/r08N4IFLsOdjpx7oYyXkZ+bDME/N3M5jEKBdi5cAbpfdS/D5IOBo8jjXBXzyyDJMSq0wetwtj5zF61wD7qeajARDxS3Oa6pfgLL02SG2hqOaZJUO9DzDVMd9LA6W6mH1ecMmylNMouCbcgpXY/ApnSmgLmb1ZO2Ab9t2WRmBxlQxod9ApeYYjG4nPMwH1/xVGoXNmw6bIwjZaPK/xnes9z5NDJOnXeCjaa90= ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-07T14:32:31.306 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:31 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/381969305' entity='client.admin' 2024-04-07T14:32:31.374 INFO:teuthology.orchestra.run.smithi083.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDs2zFtKc7hISMkBZWjtnKJt/Yhz+Ix6drlvV4GL+G7UJqGv84/+fFqEnEMelQluxqPwsbflf2FOHuC3m+sKqlKjnAcjNn4SofvmQRVQlAGON/XtaS/RUO/rU9i+TKdYaO/Eo8PTzXt8zrH4yPIdJRCbHCjSdfhJHeYtfaF/GGD1M0TZMjY6Bkt9ic7Vql1JXMtH46N7TVOnW37IB29i0P5BKoHTCcLg/OivlBqLeQmm/JejtCiGSszNSiwHXNZKt++OygCBVPmKt14kzFSfCCbOvf41f/r08N4IFLsOdjpx7oYyXkZ+bDME/N3M5jEKBdi5cAbpfdS/D5IOBo8jjXBXzyyDJMSq0wetwtj5zF61wD7qeajARDxS3Oa6pfgLL02SG2hqOaZJUO9DzDVMd9LA6W6mH1ecMmylNMouCbcgpXY/ApnSmgLmb1ZO2Ab9t2WRmBxlQxod9ApeYYjG4nPMwH1/xVGoXNmw6bIwjZaPK/xnes9z5NDJOnXeCjaa90= ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4 2024-04-07T14:32:31.532 DEBUG:teuthology.orchestra.run.smithi189:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDs2zFtKc7hISMkBZWjtnKJt/Yhz+Ix6drlvV4GL+G7UJqGv84/+fFqEnEMelQluxqPwsbflf2FOHuC3m+sKqlKjnAcjNn4SofvmQRVQlAGON/XtaS/RUO/rU9i+TKdYaO/Eo8PTzXt8zrH4yPIdJRCbHCjSdfhJHeYtfaF/GGD1M0TZMjY6Bkt9ic7Vql1JXMtH46N7TVOnW37IB29i0P5BKoHTCcLg/OivlBqLeQmm/JejtCiGSszNSiwHXNZKt++OygCBVPmKt14kzFSfCCbOvf41f/r08N4IFLsOdjpx7oYyXkZ+bDME/N3M5jEKBdi5cAbpfdS/D5IOBo8jjXBXzyyDJMSq0wetwtj5zF61wD7qeajARDxS3Oa6pfgLL02SG2hqOaZJUO9DzDVMd9LA6W6mH1ecMmylNMouCbcgpXY/ApnSmgLmb1ZO2Ab9t2WRmBxlQxod9ApeYYjG4nPMwH1/xVGoXNmw6bIwjZaPK/xnes9z5NDJOnXeCjaa90= ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-07T14:32:31.581 INFO:teuthology.orchestra.run.smithi189.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDs2zFtKc7hISMkBZWjtnKJt/Yhz+Ix6drlvV4GL+G7UJqGv84/+fFqEnEMelQluxqPwsbflf2FOHuC3m+sKqlKjnAcjNn4SofvmQRVQlAGON/XtaS/RUO/rU9i+TKdYaO/Eo8PTzXt8zrH4yPIdJRCbHCjSdfhJHeYtfaF/GGD1M0TZMjY6Bkt9ic7Vql1JXMtH46N7TVOnW37IB29i0P5BKoHTCcLg/OivlBqLeQmm/JejtCiGSszNSiwHXNZKt++OygCBVPmKt14kzFSfCCbOvf41f/r08N4IFLsOdjpx7oYyXkZ+bDME/N3M5jEKBdi5cAbpfdS/D5IOBo8jjXBXzyyDJMSq0wetwtj5zF61wD7qeajARDxS3Oa6pfgLL02SG2hqOaZJUO9DzDVMd9LA6W6mH1ecMmylNMouCbcgpXY/ApnSmgLmb1ZO2Ab9t2WRmBxlQxod9ApeYYjG4nPMwH1/xVGoXNmw6bIwjZaPK/xnes9z5NDJOnXeCjaa90= ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4 2024-04-07T14:32:31.596 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-04-07T14:32:32.125 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:31 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:32:31] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:32:32.414 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:32 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:32:32.415 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:32 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:32:32.415 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:32 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:32:32.415 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:32 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:32:32.415 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:32 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:32:32.415 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:32 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:32:32.415 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:32 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:32:36.086 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:35 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:32:35] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:32:36.086 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:35 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:32:35] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:32:36.916 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-04-07T14:32:36.917 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-04-07T14:32:37.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:36 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:32:37.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:36 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:32:37.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:36 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:32:37.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:36 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/31591119' entity='client.admin' 2024-04-07T14:32:37.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:36 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:32:37.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:36 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:32:39.475 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi189 2024-04-07T14:32:39.476 DEBUG:teuthology.orchestra.run.smithi189:> set -ex 2024-04-07T14:32:39.476 DEBUG:teuthology.orchestra.run.smithi189:> dd of=/etc/ceph/ceph.conf 2024-04-07T14:32:39.494 DEBUG:teuthology.orchestra.run.smithi189:> set -ex 2024-04-07T14:32:39.494 DEBUG:teuthology.orchestra.run.smithi189:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-07T14:32:39.552 INFO:tasks.cephadm:Adding host smithi189 to orchestrator... 2024-04-07T14:32:39.552 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph orch host add smithi189 2024-04-07T14:32:40.107 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:39 smithi083 ceph-mon[24810]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:32:40.107 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:39 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:32:40.107 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:39 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:32:40.107 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:39 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:32:40.107 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:39 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:32:40.107 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:39 smithi083 ceph-mon[24810]: Updating smithi083:/etc/ceph/ceph.conf 2024-04-07T14:32:40.107 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:39 smithi083 ceph-mon[24810]: Updating smithi083:/etc/ceph/ceph.client.admin.keyring 2024-04-07T14:32:40.107 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:39 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:32:40.107 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:39 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:32:42.596 INFO:teuthology.orchestra.run.smithi083.stdout:Added host 'smithi189' with addr '172.21.15.189' 2024-04-07T14:32:43.165 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph orch host ls --format=json 2024-04-07T14:32:43.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:43 smithi083 ceph-mon[24810]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi189", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:32:43.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:43 smithi083 ceph-mon[24810]: Deploying cephadm binary to smithi189 2024-04-07T14:32:43.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:43 smithi083 ceph-mon[24810]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:32:43.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:43 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:32:43.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:43 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:32:43.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:43 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:32:43.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:43 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:32:43.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:43 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:32:44.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:44 smithi083 ceph-mon[24810]: Added host smithi189 2024-04-07T14:32:44.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:44 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:32:44.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:44 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:32:44.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:44 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi189", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:32:44.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:44 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:32:44.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:44 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.agent.smithi189", "caps": []}]: dispatch 2024-04-07T14:32:44.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:44 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.agent.smithi189", "caps": []}]': finished 2024-04-07T14:32:45.176 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:44 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:32:44] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:32:45.178 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-07T14:32:45.178 INFO:teuthology.orchestra.run.smithi083.stdout:[{"addr": "172.21.15.83", "hostname": "smithi083", "labels": [], "status": ""}, {"addr": "172.21.15.189", "hostname": "smithi189", "labels": [], "status": ""}] 2024-04-07T14:32:45.458 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:45 smithi083 ceph-mon[24810]: Deploying daemon agent.smithi189 on smithi189 2024-04-07T14:32:45.459 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:45 smithi083 ceph-mon[24810]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:32:45.459 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:45 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:32:45.459 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:45 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:32:45.459 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:45 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:32:45.459 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:45 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:32:45.459 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:45 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:32:45.459 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:45 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:32:45.459 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:45 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:32:45.459 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:45 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:32:45.459 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:45 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:32:45.459 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:45 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:32:45.459 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:45 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:32:45.755 INFO:tasks.cephadm:Setting crush tunables to default 2024-04-07T14:32:45.755 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph osd crush tunables default 2024-04-07T14:32:46.058 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:45 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:32:45] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:32:46.369 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:46 smithi083 ceph-mon[24810]: Updating smithi189:/etc/ceph/ceph.conf 2024-04-07T14:32:46.369 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:46 smithi083 ceph-mon[24810]: Updating smithi189:/etc/ceph/ceph.client.admin.keyring 2024-04-07T14:32:46.369 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:46 smithi083 ceph-mon[24810]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-07T14:32:46.369 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:46 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:32:46.370 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:46 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:32:47.676 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:47 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:32:47] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:32:47.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:47 smithi083 ceph-mon[24810]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:32:47.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:47 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:32:47.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:47 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:32:47.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:47 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:32:48.204 INFO:teuthology.orchestra.run.smithi083.stderr:adjusted tunables profile to default 2024-04-07T14:32:48.554 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:48 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/732111135' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-07T14:32:48.822 INFO:tasks.cephadm:Adding mon.a on smithi083 2024-04-07T14:32:48.822 INFO:tasks.cephadm:Adding mon.c on smithi083 2024-04-07T14:32:48.822 INFO:tasks.cephadm:Adding mon.b on smithi189 2024-04-07T14:32:48.822 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph orch apply mon '3;smithi083:172.21.15.83=a;smithi083:[v2:172.21.15.83:3301,v1:172.21.15.83:6790]=c;smithi189:172.21.15.189=b' 2024-04-07T14:32:49.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:49 smithi083 ceph-mon[24810]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:32:49.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:49 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/732111135' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-07T14:32:49.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:49 smithi083 ceph-mon[24810]: osdmap e4: 0 total, 0 up, 0 in 2024-04-07T14:32:50.504 INFO:teuthology.orchestra.run.smithi189.stdout:Scheduled mon update... 2024-04-07T14:32:51.030 DEBUG:teuthology.orchestra.run.smithi083:mon.c> sudo journalctl -f -n 0 -u ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@mon.c.service 2024-04-07T14:32:51.034 DEBUG:teuthology.orchestra.run.smithi189:mon.b> sudo journalctl -f -n 0 -u ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@mon.b.service 2024-04-07T14:32:51.037 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-07T14:32:51.037 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph mon dump -f json 2024-04-07T14:32:51.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:51 smithi083 ceph-mon[24810]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:32:51.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:51 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:32:51.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:51 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:32:51.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:51 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:32:51.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:51 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:32:51.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:51 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:32:51.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:51 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-07T14:32:51.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:51 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:32:52.405 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:52 smithi083 ceph-mon[24810]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi083:172.21.15.83=a;smithi083:[v2:172.21.15.83:3301,v1:172.21.15.83:6790]=c;smithi189:172.21.15.189=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:32:52.405 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:52 smithi083 ceph-mon[24810]: Saving service mon spec with placement smithi083:172.21.15.83=a;smithi083:[v2:172.21.15.83:3301,v1:172.21.15.83:6790]=c;smithi189:172.21.15.189=b;count:3 2024-04-07T14:32:52.405 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:52 smithi083 ceph-mon[24810]: Deploying daemon mon.c on smithi083 2024-04-07T14:32:52.917 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-04-07T14:32:52.917 INFO:teuthology.orchestra.run.smithi189.stdout:{"epoch":1,"fsid":"7388281a-f4eb-11ee-b649-cb9ed24678a4","modified":"2024-04-07T14:31:28.118696Z","created":"2024-04-07T14:31:28.118696Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-07T14:32:52.919 INFO:teuthology.orchestra.run.smithi189.stderr:dumped monmap epoch 1 2024-04-07T14:32:53.262 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:53 smithi083 ceph-mon[24810]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:32:53.262 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:53 smithi083 ceph-mon[24810]: from='client.? 172.21.15.189:0/3617341993' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-07T14:32:54.490 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-07T14:32:54.491 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph mon dump -f json 2024-04-07T14:33:00.426 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:32:59 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:32:59] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:33:00.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:59 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-07T14:33:00.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:59 smithi083 ceph-mon[24810]: mon.a calling monitor election 2024-04-07T14:33:00.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:59 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-07T14:33:00.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:59 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-07T14:33:00.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:59 smithi083 ceph-mon[24810]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:00.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:59 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-07T14:33:00.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:59 smithi083 ceph-mon[24810]: mon.c calling monitor election 2024-04-07T14:33:00.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:59 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-07T14:33:00.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:59 smithi083 ceph-mon[24810]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:00.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:59 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-07T14:33:00.428 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:59 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-07T14:33:00.428 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:59 smithi083 ceph-mon[24810]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-07T14:33:00.428 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:59 smithi083 ceph-mon[24810]: monmap e2: 2 mons at {a=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0],c=[v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0]} 2024-04-07T14:33:00.428 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:59 smithi083 ceph-mon[24810]: fsmap 2024-04-07T14:33:00.428 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:59 smithi083 ceph-mon[24810]: osdmap e4: 0 total, 0 up, 0 in 2024-04-07T14:33:00.428 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:59 smithi083 ceph-mon[24810]: mgrmap e13: y(active, since 37s) 2024-04-07T14:33:00.428 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:59 smithi083 ceph-mon[24810]: overall HEALTH_OK 2024-04-07T14:33:00.428 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:59 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:00.428 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:59 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:00.428 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:59 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-07T14:33:00.428 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:59 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:00.428 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:32:59 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:00.429 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:32:59 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:00.613 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-04-07T14:33:00.613 INFO:teuthology.orchestra.run.smithi189.stdout:{"epoch":2,"fsid":"7388281a-f4eb-11ee-b649-cb9ed24678a4","modified":"2024-04-07T14:32:54.903447Z","created":"2024-04-07T14:31:28.118696Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3301","nonce":0},{"type":"v1","addr":"172.21.15.83:6790","nonce":0}]},"addr":"172.21.15.83:6790/0","public_addr":"172.21.15.83:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-07T14:33:00.616 INFO:teuthology.orchestra.run.smithi189.stderr:dumped monmap epoch 2 2024-04-07T14:33:01.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:01 smithi083 ceph-mon[24810]: Deploying daemon mon.b on smithi189 2024-04-07T14:33:01.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:01 smithi083 ceph-mon[24810]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:01.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:01 smithi083 ceph-mon[24810]: from='client.? 172.21.15.189:0/4154561844' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-07T14:33:01.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:01 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-07T14:33:01.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:01 smithi083 ceph-mon[29774]: Deploying daemon mon.b on smithi189 2024-04-07T14:33:01.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:01 smithi083 ceph-mon[29774]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:01.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:01 smithi083 ceph-mon[29774]: from='client.? 172.21.15.189:0/4154561844' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-07T14:33:01.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:01 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-07T14:33:02.867 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-07T14:33:02.867 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph mon dump -f json 2024-04-07T14:33:03.478 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:03 smithi083 ceph-mon[24810]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:03.479 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:03 smithi083 ceph-mon[29774]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:04.794 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-04-07T14:33:04.795 INFO:teuthology.orchestra.run.smithi189.stdout:{"epoch":2,"fsid":"7388281a-f4eb-11ee-b649-cb9ed24678a4","modified":"2024-04-07T14:32:54.903447Z","created":"2024-04-07T14:31:28.118696Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3301","nonce":0},{"type":"v1","addr":"172.21.15.83:6790","nonce":0}]},"addr":"172.21.15.83:6790/0","public_addr":"172.21.15.83:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-07T14:33:04.796 INFO:teuthology.orchestra.run.smithi189.stderr:dumped monmap epoch 2 2024-04-07T14:33:07.468 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-07T14:33:07.469 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph mon dump -f json 2024-04-07T14:33:10.163 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:33:09] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:33:10.163 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:33:10 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:33:10] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:33:10.164 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-07T14:33:10.164 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[29774]: mon.a calling monitor election 2024-04-07T14:33:10.164 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[29774]: mon.c calling monitor election 2024-04-07T14:33:10.164 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-07T14:33:10.164 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-07T14:33:10.164 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-07T14:33:10.164 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[29774]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:10.164 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-07T14:33:10.164 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-07T14:33:10.164 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[29774]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:10.165 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-07T14:33:10.165 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-07T14:33:10.165 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[29774]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-07T14:33:10.165 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[29774]: monmap e3: 3 mons at {a=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0],b=[v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0],c=[v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0]} 2024-04-07T14:33:10.165 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[29774]: fsmap 2024-04-07T14:33:10.165 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[29774]: osdmap e4: 0 total, 0 up, 0 in 2024-04-07T14:33:10.165 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[29774]: mgrmap e13: y(active, since 47s) 2024-04-07T14:33:10.165 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[29774]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-04-07T14:33:10.165 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[29774]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-04-07T14:33:10.165 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[29774]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-04-07T14:33:10.165 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[29774]: mon.b (rank 2) addr [v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0] is down (out of quorum) 2024-04-07T14:33:10.165 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:10.165 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:10.165 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:33:10.165 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:10.165 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:10.165 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:33:10.166 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-07T14:33:10.166 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[24810]: mon.a calling monitor election 2024-04-07T14:33:10.166 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[24810]: mon.c calling monitor election 2024-04-07T14:33:10.166 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-07T14:33:10.166 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-07T14:33:10.166 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-07T14:33:10.166 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[24810]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:10.166 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-07T14:33:10.166 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-07T14:33:10.166 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[24810]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:10.166 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-07T14:33:10.166 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-07T14:33:10.166 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[24810]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-07T14:33:10.167 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[24810]: monmap e3: 3 mons at {a=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0],b=[v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0],c=[v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0]} 2024-04-07T14:33:10.167 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[24810]: fsmap 2024-04-07T14:33:10.167 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[24810]: osdmap e4: 0 total, 0 up, 0 in 2024-04-07T14:33:10.167 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[24810]: mgrmap e13: y(active, since 47s) 2024-04-07T14:33:10.167 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[24810]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-04-07T14:33:10.167 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[24810]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-04-07T14:33:10.167 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[24810]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-04-07T14:33:10.167 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[24810]: mon.b (rank 2) addr [v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0] is down (out of quorum) 2024-04-07T14:33:10.167 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:10.167 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:10.167 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:33:10.167 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:10.167 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:10.167 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:09 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:33:10.308 INFO:teuthology.orchestra.run.smithi189.stdout: 2024-04-07T14:33:10.308 INFO:teuthology.orchestra.run.smithi189.stdout:{"epoch":3,"fsid":"7388281a-f4eb-11ee-b649-cb9ed24678a4","modified":"2024-04-07T14:33:04.866265Z","created":"2024-04-07T14:31:28.118696Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3300","nonce":0},{"type":"v1","addr":"172.21.15.83:6789","nonce":0}]},"addr":"172.21.15.83:6789/0","public_addr":"172.21.15.83:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:3301","nonce":0},{"type":"v1","addr":"172.21.15.83:6790","nonce":0}]},"addr":"172.21.15.83:6790/0","public_addr":"172.21.15.83:6790/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:3300","nonce":0},{"type":"v1","addr":"172.21.15.189:6789","nonce":0}]},"addr":"172.21.15.189:6789/0","public_addr":"172.21.15.189:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-07T14:33:10.310 INFO:teuthology.orchestra.run.smithi189.stderr:dumped monmap epoch 3 2024-04-07T14:33:10.905 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-04-07T14:33:10.905 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph config generate-minimal-conf 2024-04-07T14:33:11.176 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:33:11 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:33:11] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:33:11.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:10 smithi083 ceph-mon[29774]: Updating smithi083:/etc/ceph/ceph.conf 2024-04-07T14:33:11.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:10 smithi083 ceph-mon[29774]: Updating smithi189:/etc/ceph/ceph.conf 2024-04-07T14:33:11.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:10 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:11.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:10 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:11.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:10 smithi083 ceph-mon[29774]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:11.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:10 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:11.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:11 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:11.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:11 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:11.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:11 smithi083 ceph-mon[29774]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:33:11.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:11 smithi083 ceph-mon[29774]: Reconfiguring mon.a (unknown last config time)... 2024-04-07T14:33:11.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:11 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-07T14:33:11.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:11 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-07T14:33:11.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:11 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:11.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:11 smithi083 ceph-mon[29774]: Reconfiguring daemon mon.a on smithi083 2024-04-07T14:33:11.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:11 smithi083 ceph-mon[29774]: from='client.? 172.21.15.189:0/918940996' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-07T14:33:11.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:11 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-07T14:33:11.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:10 smithi083 ceph-mon[24810]: Updating smithi083:/etc/ceph/ceph.conf 2024-04-07T14:33:11.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:10 smithi083 ceph-mon[24810]: Updating smithi189:/etc/ceph/ceph.conf 2024-04-07T14:33:11.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:10 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:11.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:10 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:11.178 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:10 smithi083 ceph-mon[24810]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:11.178 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:10 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:11.178 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:10 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:11.178 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:10 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:11.178 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:10 smithi083 ceph-mon[24810]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:33:11.178 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:10 smithi083 ceph-mon[24810]: Reconfiguring mon.a (unknown last config time)... 2024-04-07T14:33:11.178 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:10 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-07T14:33:11.178 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:11 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-07T14:33:11.178 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:11 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:11.178 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:11 smithi083 ceph-mon[24810]: Reconfiguring daemon mon.a on smithi083 2024-04-07T14:33:11.178 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:11 smithi083 ceph-mon[24810]: from='client.? 172.21.15.189:0/918940996' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-07T14:33:11.178 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:11 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-07T14:33:12.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:12 smithi083 ceph-mon[24810]: mon.b calling monitor election 2024-04-07T14:33:12.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:12 smithi083 ceph-mon[24810]: Reconfiguring mon.c (monmap changed)... 2024-04-07T14:33:12.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:12 smithi083 ceph-mon[24810]: Reconfiguring daemon mon.c on smithi083 2024-04-07T14:33:12.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:12 smithi083 ceph-mon[24810]: mon.b calling monitor election 2024-04-07T14:33:12.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:12 smithi083 ceph-mon[24810]: mon.c calling monitor election 2024-04-07T14:33:12.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:12 smithi083 ceph-mon[24810]: mon.a calling monitor election 2024-04-07T14:33:12.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:12 smithi083 ceph-mon[24810]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-07T14:33:12.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:12 smithi083 ceph-mon[24810]: monmap e3: 3 mons at {a=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0],b=[v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0],c=[v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0]} 2024-04-07T14:33:12.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:12 smithi083 ceph-mon[24810]: fsmap 2024-04-07T14:33:12.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:12 smithi083 ceph-mon[24810]: osdmap e4: 0 total, 0 up, 0 in 2024-04-07T14:33:12.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:12 smithi083 ceph-mon[24810]: mgrmap e13: y(active, since 49s) 2024-04-07T14:33:12.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:12 smithi083 ceph-mon[24810]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-04-07T14:33:12.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:12 smithi083 ceph-mon[24810]: Cluster is now healthy 2024-04-07T14:33:12.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:12 smithi083 ceph-mon[24810]: overall HEALTH_OK 2024-04-07T14:33:12.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:12 smithi083 ceph-mon[24810]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:12.928 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:12 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:12.928 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:12 smithi083 ceph-mon[29774]: mon.b calling monitor election 2024-04-07T14:33:12.928 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:12 smithi083 ceph-mon[29774]: Reconfiguring mon.c (monmap changed)... 2024-04-07T14:33:12.928 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:12 smithi083 ceph-mon[29774]: Reconfiguring daemon mon.c on smithi083 2024-04-07T14:33:12.928 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:12 smithi083 ceph-mon[29774]: mon.b calling monitor election 2024-04-07T14:33:12.928 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:12 smithi083 ceph-mon[29774]: mon.c calling monitor election 2024-04-07T14:33:12.928 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:12 smithi083 ceph-mon[29774]: mon.a calling monitor election 2024-04-07T14:33:12.928 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:12 smithi083 ceph-mon[29774]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-07T14:33:12.928 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:12 smithi083 ceph-mon[29774]: monmap e3: 3 mons at {a=[v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0],b=[v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0],c=[v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0]} 2024-04-07T14:33:12.928 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:12 smithi083 ceph-mon[29774]: fsmap 2024-04-07T14:33:12.928 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:12 smithi083 ceph-mon[29774]: osdmap e4: 0 total, 0 up, 0 in 2024-04-07T14:33:12.928 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:12 smithi083 ceph-mon[29774]: mgrmap e13: y(active, since 49s) 2024-04-07T14:33:12.929 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:12 smithi083 ceph-mon[29774]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-04-07T14:33:12.929 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:12 smithi083 ceph-mon[29774]: Cluster is now healthy 2024-04-07T14:33:12.929 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:12 smithi083 ceph-mon[29774]: overall HEALTH_OK 2024-04-07T14:33:12.929 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:12 smithi083 ceph-mon[29774]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:12.929 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:12 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:13.580 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:33:13 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:33:13] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:33:13.580 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:33:13 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:33:13] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:33:13.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:13 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:13.911 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:13 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-07T14:33:13.911 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:13 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:13.912 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:13 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:13.912 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:13 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:13.912 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:13 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:13.912 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:13 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-07T14:33:13.912 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:13 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:13.912 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:13 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:13.912 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:13 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:13.912 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:13 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:14.197 INFO:teuthology.orchestra.run.smithi083.stdout:# minimal ceph.conf for 7388281a-f4eb-11ee-b649-cb9ed24678a4 2024-04-07T14:33:14.198 INFO:teuthology.orchestra.run.smithi083.stdout:[global] 2024-04-07T14:33:14.198 INFO:teuthology.orchestra.run.smithi083.stdout: fsid = 7388281a-f4eb-11ee-b649-cb9ed24678a4 2024-04-07T14:33:14.198 INFO:teuthology.orchestra.run.smithi083.stdout: mon_host = [v2:172.21.15.83:3300/0,v1:172.21.15.83:6789/0] [v2:172.21.15.189:3300/0,v1:172.21.15.189:6789/0] [v2:172.21.15.83:3301/0,v1:172.21.15.83:6790/0] 2024-04-07T14:33:14.822 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-04-07T14:33:14.822 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-04-07T14:33:14.822 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd of=/etc/ceph/ceph.conf 2024-04-07T14:33:14.848 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:14 smithi083 ceph-mon[24810]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:14.848 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:14 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/805147566' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:14.848 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:14 smithi083 ceph-mon[29774]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:14.848 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:14 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/805147566' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:14.852 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-04-07T14:33:14.852 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-07T14:33:14.919 DEBUG:teuthology.orchestra.run.smithi189:> set -ex 2024-04-07T14:33:14.919 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd of=/etc/ceph/ceph.conf 2024-04-07T14:33:14.951 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:14 smithi189 ceph-mon[25797]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:14.951 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:14 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/805147566' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:14.956 DEBUG:teuthology.orchestra.run.smithi189:> set -ex 2024-04-07T14:33:14.956 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-07T14:33:15.028 INFO:tasks.cephadm:Adding mgr.y on smithi083 2024-04-07T14:33:15.028 INFO:tasks.cephadm:Adding mgr.x on smithi189 2024-04-07T14:33:15.028 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph orch apply mgr '2;smithi083=y;smithi189=x' 2024-04-07T14:33:16.544 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:33:16 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:33:16] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:33:16.907 INFO:teuthology.orchestra.run.smithi189.stdout:Scheduled mgr update... 2024-04-07T14:33:17.450 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:17 smithi189 ceph-mon[25797]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:17.450 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:17 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:17.450 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:17 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:17.450 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:17 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:17.475 DEBUG:teuthology.orchestra.run.smithi189:mgr.x> sudo journalctl -f -n 0 -u ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@mgr.x.service 2024-04-07T14:33:17.477 INFO:tasks.cephadm:Deploying OSDs... 2024-04-07T14:33:17.477 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-04-07T14:33:17.477 DEBUG:teuthology.orchestra.run.smithi083:> dd if=/scratch_devs of=/dev/stdout 2024-04-07T14:33:17.493 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-07T14:33:17.493 DEBUG:teuthology.orchestra.run.smithi083:> stat /dev/vg_nvme/lv_1 2024-04-07T14:33:17.548 INFO:teuthology.orchestra.run.smithi083.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-07T14:33:17.548 INFO:teuthology.orchestra.run.smithi083.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-07T14:33:17.548 INFO:teuthology.orchestra.run.smithi083.stdout:Device: 5h/5d Inode: 668 Links: 1 2024-04-07T14:33:17.548 INFO:teuthology.orchestra.run.smithi083.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-07T14:33:17.548 INFO:teuthology.orchestra.run.smithi083.stdout:Context: system_u:object_r:device_t:s0 2024-04-07T14:33:17.548 INFO:teuthology.orchestra.run.smithi083.stdout:Access: 2024-04-07 14:33:15.636307297 +0000 2024-04-07T14:33:17.548 INFO:teuthology.orchestra.run.smithi083.stdout:Modify: 2024-04-07 14:30:32.731442468 +0000 2024-04-07T14:33:17.548 INFO:teuthology.orchestra.run.smithi083.stdout:Change: 2024-04-07 14:30:32.731442468 +0000 2024-04-07T14:33:17.549 INFO:teuthology.orchestra.run.smithi083.stdout: Birth: 2024-04-07 14:30:32.731442468 +0000 2024-04-07T14:33:17.549 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-07T14:33:17.612 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:17 smithi083 ceph-mon[29774]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:17.612 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:17 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:17.612 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:17 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:17.612 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:17 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:17.612 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:17 smithi083 ceph-mon[24810]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:17.612 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:17 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:17.612 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:17 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:17.613 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:17 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:17.615 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records in 2024-04-07T14:33:17.615 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records out 2024-04-07T14:33:17.615 INFO:teuthology.orchestra.run.smithi083.stderr:512 bytes copied, 0.000182758 s, 2.8 MB/s 2024-04-07T14:33:17.616 DEBUG:teuthology.orchestra.run.smithi083:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-07T14:33:17.671 DEBUG:teuthology.orchestra.run.smithi083:> stat /dev/vg_nvme/lv_2 2024-04-07T14:33:17.726 INFO:teuthology.orchestra.run.smithi083.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-07T14:33:17.726 INFO:teuthology.orchestra.run.smithi083.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-07T14:33:17.726 INFO:teuthology.orchestra.run.smithi083.stdout:Device: 5h/5d Inode: 687 Links: 1 2024-04-07T14:33:17.726 INFO:teuthology.orchestra.run.smithi083.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-07T14:33:17.726 INFO:teuthology.orchestra.run.smithi083.stdout:Context: system_u:object_r:device_t:s0 2024-04-07T14:33:17.726 INFO:teuthology.orchestra.run.smithi083.stdout:Access: 2024-04-07 14:33:15.636307297 +0000 2024-04-07T14:33:17.726 INFO:teuthology.orchestra.run.smithi083.stdout:Modify: 2024-04-07 14:30:33.077433841 +0000 2024-04-07T14:33:17.726 INFO:teuthology.orchestra.run.smithi083.stdout:Change: 2024-04-07 14:30:33.077433841 +0000 2024-04-07T14:33:17.726 INFO:teuthology.orchestra.run.smithi083.stdout: Birth: 2024-04-07 14:30:33.077433841 +0000 2024-04-07T14:33:17.726 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-07T14:33:17.792 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records in 2024-04-07T14:33:17.792 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records out 2024-04-07T14:33:17.792 INFO:teuthology.orchestra.run.smithi083.stderr:512 bytes copied, 0.000211613 s, 2.4 MB/s 2024-04-07T14:33:17.793 DEBUG:teuthology.orchestra.run.smithi083:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-07T14:33:17.848 DEBUG:teuthology.orchestra.run.smithi083:> stat /dev/vg_nvme/lv_3 2024-04-07T14:33:17.903 INFO:teuthology.orchestra.run.smithi083.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-07T14:33:17.903 INFO:teuthology.orchestra.run.smithi083.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-07T14:33:17.903 INFO:teuthology.orchestra.run.smithi083.stdout:Device: 5h/5d Inode: 706 Links: 1 2024-04-07T14:33:17.903 INFO:teuthology.orchestra.run.smithi083.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-07T14:33:17.903 INFO:teuthology.orchestra.run.smithi083.stdout:Context: system_u:object_r:device_t:s0 2024-04-07T14:33:17.903 INFO:teuthology.orchestra.run.smithi083.stdout:Access: 2024-04-07 14:33:15.636307297 +0000 2024-04-07T14:33:17.904 INFO:teuthology.orchestra.run.smithi083.stdout:Modify: 2024-04-07 14:30:33.410425539 +0000 2024-04-07T14:33:17.904 INFO:teuthology.orchestra.run.smithi083.stdout:Change: 2024-04-07 14:30:33.410425539 +0000 2024-04-07T14:33:17.904 INFO:teuthology.orchestra.run.smithi083.stdout: Birth: 2024-04-07 14:30:33.410425539 +0000 2024-04-07T14:33:17.904 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-07T14:33:17.968 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records in 2024-04-07T14:33:17.969 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records out 2024-04-07T14:33:17.969 INFO:teuthology.orchestra.run.smithi083.stderr:512 bytes copied, 0.00021329 s, 2.4 MB/s 2024-04-07T14:33:17.970 DEBUG:teuthology.orchestra.run.smithi083:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-07T14:33:18.025 DEBUG:teuthology.orchestra.run.smithi083:> stat /dev/vg_nvme/lv_4 2024-04-07T14:33:18.080 INFO:teuthology.orchestra.run.smithi083.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-07T14:33:18.080 INFO:teuthology.orchestra.run.smithi083.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-07T14:33:18.080 INFO:teuthology.orchestra.run.smithi083.stdout:Device: 5h/5d Inode: 725 Links: 1 2024-04-07T14:33:18.080 INFO:teuthology.orchestra.run.smithi083.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-07T14:33:18.080 INFO:teuthology.orchestra.run.smithi083.stdout:Context: system_u:object_r:device_t:s0 2024-04-07T14:33:18.080 INFO:teuthology.orchestra.run.smithi083.stdout:Access: 2024-04-07 14:33:15.636307297 +0000 2024-04-07T14:33:18.080 INFO:teuthology.orchestra.run.smithi083.stdout:Modify: 2024-04-07 14:30:33.746417163 +0000 2024-04-07T14:33:18.080 INFO:teuthology.orchestra.run.smithi083.stdout:Change: 2024-04-07 14:30:33.746417163 +0000 2024-04-07T14:33:18.080 INFO:teuthology.orchestra.run.smithi083.stdout: Birth: 2024-04-07 14:30:33.746417163 +0000 2024-04-07T14:33:18.080 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-07T14:33:18.145 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records in 2024-04-07T14:33:18.146 INFO:teuthology.orchestra.run.smithi083.stderr:1+0 records out 2024-04-07T14:33:18.146 INFO:teuthology.orchestra.run.smithi083.stderr:512 bytes copied, 0.000229423 s, 2.2 MB/s 2024-04-07T14:33:18.147 DEBUG:teuthology.orchestra.run.smithi083:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-07T14:33:18.203 DEBUG:teuthology.orchestra.run.smithi189:> set -ex 2024-04-07T14:33:18.203 DEBUG:teuthology.orchestra.run.smithi189:> dd if=/scratch_devs of=/dev/stdout 2024-04-07T14:33:18.219 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-07T14:33:18.219 DEBUG:teuthology.orchestra.run.smithi189:> stat /dev/vg_nvme/lv_1 2024-04-07T14:33:18.275 INFO:teuthology.orchestra.run.smithi189.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-07T14:33:18.275 INFO:teuthology.orchestra.run.smithi189.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-07T14:33:18.275 INFO:teuthology.orchestra.run.smithi189.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-04-07T14:33:18.275 INFO:teuthology.orchestra.run.smithi189.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-07T14:33:18.275 INFO:teuthology.orchestra.run.smithi189.stdout:Context: system_u:object_r:device_t:s0 2024-04-07T14:33:18.275 INFO:teuthology.orchestra.run.smithi189.stdout:Access: 2024-04-07 14:33:12.800134485 +0000 2024-04-07T14:33:18.275 INFO:teuthology.orchestra.run.smithi189.stdout:Modify: 2024-04-07 14:30:32.746617560 +0000 2024-04-07T14:33:18.275 INFO:teuthology.orchestra.run.smithi189.stdout:Change: 2024-04-07 14:30:32.746617560 +0000 2024-04-07T14:33:18.275 INFO:teuthology.orchestra.run.smithi189.stdout: Birth: 2024-04-07 14:30:32.746617560 +0000 2024-04-07T14:33:18.276 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-07T14:33:18.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:18 smithi189 ceph-mon[25797]: from='client.24104 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi083=y;smithi189=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:33:18.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:18 smithi189 ceph-mon[25797]: Saving service mgr spec with placement smithi083=y;smithi189=x;count:2 2024-04-07T14:33:18.339 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:18 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:18.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:18 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-07T14:33:18.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:18 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-07T14:33:18.340 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:18 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:18.342 INFO:teuthology.orchestra.run.smithi189.stderr:1+0 records in 2024-04-07T14:33:18.342 INFO:teuthology.orchestra.run.smithi189.stderr:1+0 records out 2024-04-07T14:33:18.342 INFO:teuthology.orchestra.run.smithi189.stderr:512 bytes copied, 0.000213018 s, 2.4 MB/s 2024-04-07T14:33:18.343 DEBUG:teuthology.orchestra.run.smithi189:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-07T14:33:18.399 DEBUG:teuthology.orchestra.run.smithi189:> stat /dev/vg_nvme/lv_2 2024-04-07T14:33:18.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:18 smithi083 ceph-mon[29774]: from='client.24104 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi083=y;smithi189=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:33:18.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:18 smithi083 ceph-mon[29774]: Saving service mgr spec with placement smithi083=y;smithi189=x;count:2 2024-04-07T14:33:18.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:18 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:18.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:18 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-07T14:33:18.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:18 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-07T14:33:18.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:18 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:18.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:18 smithi083 ceph-mon[24810]: from='client.24104 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi083=y;smithi189=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:33:18.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:18 smithi083 ceph-mon[24810]: Saving service mgr spec with placement smithi083=y;smithi189=x;count:2 2024-04-07T14:33:18.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:18 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:18.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:18 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-07T14:33:18.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:18 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-07T14:33:18.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:18 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:18.453 INFO:teuthology.orchestra.run.smithi189.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-07T14:33:18.453 INFO:teuthology.orchestra.run.smithi189.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-07T14:33:18.453 INFO:teuthology.orchestra.run.smithi189.stdout:Device: 5h/5d Inode: 687 Links: 1 2024-04-07T14:33:18.453 INFO:teuthology.orchestra.run.smithi189.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-07T14:33:18.453 INFO:teuthology.orchestra.run.smithi189.stdout:Context: system_u:object_r:device_t:s0 2024-04-07T14:33:18.453 INFO:teuthology.orchestra.run.smithi189.stdout:Access: 2024-04-07 14:33:12.800134485 +0000 2024-04-07T14:33:18.454 INFO:teuthology.orchestra.run.smithi189.stdout:Modify: 2024-04-07 14:30:33.082610844 +0000 2024-04-07T14:33:18.454 INFO:teuthology.orchestra.run.smithi189.stdout:Change: 2024-04-07 14:30:33.082610844 +0000 2024-04-07T14:33:18.454 INFO:teuthology.orchestra.run.smithi189.stdout: Birth: 2024-04-07 14:30:33.082610844 +0000 2024-04-07T14:33:18.454 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-07T14:33:18.520 INFO:teuthology.orchestra.run.smithi189.stderr:1+0 records in 2024-04-07T14:33:18.520 INFO:teuthology.orchestra.run.smithi189.stderr:1+0 records out 2024-04-07T14:33:18.520 INFO:teuthology.orchestra.run.smithi189.stderr:512 bytes copied, 0.0001959 s, 2.6 MB/s 2024-04-07T14:33:18.521 DEBUG:teuthology.orchestra.run.smithi189:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-07T14:33:18.578 DEBUG:teuthology.orchestra.run.smithi189:> stat /dev/vg_nvme/lv_3 2024-04-07T14:33:18.633 INFO:teuthology.orchestra.run.smithi189.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-07T14:33:18.633 INFO:teuthology.orchestra.run.smithi189.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-07T14:33:18.633 INFO:teuthology.orchestra.run.smithi189.stdout:Device: 5h/5d Inode: 706 Links: 1 2024-04-07T14:33:18.633 INFO:teuthology.orchestra.run.smithi189.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-07T14:33:18.633 INFO:teuthology.orchestra.run.smithi189.stdout:Context: system_u:object_r:device_t:s0 2024-04-07T14:33:18.633 INFO:teuthology.orchestra.run.smithi189.stdout:Access: 2024-04-07 14:33:12.800134485 +0000 2024-04-07T14:33:18.633 INFO:teuthology.orchestra.run.smithi189.stdout:Modify: 2024-04-07 14:30:33.429603908 +0000 2024-04-07T14:33:18.633 INFO:teuthology.orchestra.run.smithi189.stdout:Change: 2024-04-07 14:30:33.429603908 +0000 2024-04-07T14:33:18.633 INFO:teuthology.orchestra.run.smithi189.stdout: Birth: 2024-04-07 14:30:33.429603908 +0000 2024-04-07T14:33:18.634 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-07T14:33:18.699 INFO:teuthology.orchestra.run.smithi189.stderr:1+0 records in 2024-04-07T14:33:18.699 INFO:teuthology.orchestra.run.smithi189.stderr:1+0 records out 2024-04-07T14:33:18.699 INFO:teuthology.orchestra.run.smithi189.stderr:512 bytes copied, 0.000202356 s, 2.5 MB/s 2024-04-07T14:33:18.700 DEBUG:teuthology.orchestra.run.smithi189:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-07T14:33:18.756 DEBUG:teuthology.orchestra.run.smithi189:> stat /dev/vg_nvme/lv_4 2024-04-07T14:33:18.811 INFO:teuthology.orchestra.run.smithi189.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-07T14:33:18.811 INFO:teuthology.orchestra.run.smithi189.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-07T14:33:18.811 INFO:teuthology.orchestra.run.smithi189.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-04-07T14:33:18.811 INFO:teuthology.orchestra.run.smithi189.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-07T14:33:18.811 INFO:teuthology.orchestra.run.smithi189.stdout:Context: system_u:object_r:device_t:s0 2024-04-07T14:33:18.811 INFO:teuthology.orchestra.run.smithi189.stdout:Access: 2024-04-07 14:33:12.800134485 +0000 2024-04-07T14:33:18.811 INFO:teuthology.orchestra.run.smithi189.stdout:Modify: 2024-04-07 14:30:33.777596952 +0000 2024-04-07T14:33:18.811 INFO:teuthology.orchestra.run.smithi189.stdout:Change: 2024-04-07 14:30:33.777596952 +0000 2024-04-07T14:33:18.811 INFO:teuthology.orchestra.run.smithi189.stdout: Birth: 2024-04-07 14:30:33.777596952 +0000 2024-04-07T14:33:18.812 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-07T14:33:18.877 INFO:teuthology.orchestra.run.smithi189.stderr:1+0 records in 2024-04-07T14:33:18.877 INFO:teuthology.orchestra.run.smithi189.stderr:1+0 records out 2024-04-07T14:33:18.877 INFO:teuthology.orchestra.run.smithi189.stderr:512 bytes copied, 0.000215416 s, 2.4 MB/s 2024-04-07T14:33:18.878 DEBUG:teuthology.orchestra.run.smithi189:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-07T14:33:18.934 INFO:tasks.cephadm:Deploying osd.0 on smithi083 with /dev/vg_nvme/lv_4... 2024-04-07T14:33:18.934 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-07T14:33:19.245 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:19 smithi083 ceph-mon[29774]: Reconfiguring mon.b (monmap changed)... 2024-04-07T14:33:19.245 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:19 smithi083 ceph-mon[29774]: Reconfiguring daemon mon.b on smithi189 2024-04-07T14:33:19.245 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:19 smithi083 ceph-mon[29774]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:19.245 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:19 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:19.245 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:19 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:33:19.245 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:19 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:19.245 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:19 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:33:19.246 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:19 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:19.246 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:19 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-07T14:33:19.246 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:19 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-07T14:33:19.246 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:19 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-07T14:33:19.246 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:19 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:19.246 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:19 smithi083 ceph-mon[24810]: Reconfiguring mon.b (monmap changed)... 2024-04-07T14:33:19.246 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:19 smithi083 ceph-mon[24810]: Reconfiguring daemon mon.b on smithi189 2024-04-07T14:33:19.247 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:19 smithi083 ceph-mon[24810]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:19.247 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:19 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:19.247 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:19 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:33:19.247 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:19 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:19.247 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:19 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:33:19.247 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:19 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:19.247 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:19 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-07T14:33:19.247 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:19 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-07T14:33:19.247 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:19 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-07T14:33:19.247 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:19 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:19.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:19 smithi189 ceph-mon[25797]: Reconfiguring mon.b (monmap changed)... 2024-04-07T14:33:19.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:19 smithi189 ceph-mon[25797]: Reconfiguring daemon mon.b on smithi189 2024-04-07T14:33:19.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:19 smithi189 ceph-mon[25797]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:19.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:19 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:19.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:19 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:33:19.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:19 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:19.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:19 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:33:19.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:19 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:19.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:19 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-07T14:33:19.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:19 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-07T14:33:19.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:19 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-07T14:33:19.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:19 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:20.439 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:20 smithi189 ceph-mon[25797]: Deploying daemon mgr.x on smithi189 2024-04-07T14:33:20.439 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:33:20 smithi189 systemd[1]: Starting Ceph mgr.x for 7388281a-f4eb-11ee-b649-cb9ed24678a4... 2024-04-07T14:33:20.489 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:20 smithi083 ceph-mon[24810]: Deploying daemon mgr.x on smithi189 2024-04-07T14:33:20.489 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:20 smithi083 ceph-mon[29774]: Deploying daemon mgr.x on smithi189 2024-04-07T14:33:20.707 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:33:20 smithi189 podman[27334]: 2024-04-07 14:33:20.343073018 +0000 UTC m=+0.018065516 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-07T14:33:20.707 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:33:20 smithi189 podman[27334]: 2024-04-07 14:33:20.464123867 +0000 UTC m=+0.139116392 container create 14d5f8aeae614d501de1af6c275a1410e9ebe8715451e0cd16abcaec5ba19b95 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, name=centos-stream, build-date=2022-05-03T08:36:31.336870, version=8, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, ceph=True, distribution-scope=public, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-07T14:33:20.707 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:33:20 smithi189 podman[27334]: 2024-04-07 14:33:20.649792362 +0000 UTC m=+0.324784844 container init 14d5f8aeae614d501de1af6c275a1410e9ebe8715451e0cd16abcaec5ba19b95 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, release=754, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, com.redhat.component=centos-stream-container, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, ceph=True, io.openshift.tags=base centos centos-stream) 2024-04-07T14:33:20.707 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:33:20 smithi189 podman[27334]: 2024-04-07 14:33:20.652210349 +0000 UTC m=+0.327202839 container start 14d5f8aeae614d501de1af6c275a1410e9ebe8715451e0cd16abcaec5ba19b95 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, ceph=True, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, version=8, RELEASE=HEAD, architecture=x86_64, name=centos-stream, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container) 2024-04-07T14:33:20.707 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:33:20 smithi189 ceph-mgr[27353]: -- 172.21.15.189:0/1568198540 <== mon.2 v2:172.21.15.189:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5652318a4340 con 0x56523187d000 2024-04-07T14:33:20.707 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:33:20 smithi189 bash[27334]: 14d5f8aeae614d501de1af6c275a1410e9ebe8715451e0cd16abcaec5ba19b95 2024-04-07T14:33:20.958 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:33:20 smithi189 systemd[1]: Started Ceph mgr.x for 7388281a-f4eb-11ee-b649-cb9ed24678a4. 2024-04-07T14:33:20.994 INFO:teuthology.orchestra.run.smithi083.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-07T14:33:20.994 INFO:teuthology.orchestra.run.smithi083.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-07T14:33:20.994 INFO:teuthology.orchestra.run.smithi083.stderr: stderr: 10+0 records in 2024-04-07T14:33:20.994 INFO:teuthology.orchestra.run.smithi083.stderr:10+0 records out 2024-04-07T14:33:20.994 INFO:teuthology.orchestra.run.smithi083.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0114496 s, 916 MB/s 2024-04-07T14:33:20.995 INFO:teuthology.orchestra.run.smithi083.stderr: stderr: 2024-04-07T14:33:20.995 INFO:teuthology.orchestra.run.smithi083.stderr:--> Zapping successful for: 2024-04-07T14:33:21.318 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:33:21 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:33:21.157+0000 7f7425309000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-07T14:33:21.318 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:33:21 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:33:21.251+0000 7f7425309000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-07T14:33:21.318 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:21 smithi189 ceph-mon[25797]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:21.318 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:21 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:21.319 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:21 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:33:21.319 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:21 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:21.319 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:21 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:33:21.319 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:21 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:21.319 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:21 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-07T14:33:21.319 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:21 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-07T14:33:21.319 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:21 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:21.567 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:21 smithi083 ceph-mon[24810]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:21.567 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:21 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:21.567 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:21 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:33:21.567 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:21 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:21.567 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:21 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:33:21.568 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:21 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:21.568 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:21 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-07T14:33:21.568 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:21 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-07T14:33:21.568 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:21 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:21.568 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:21 smithi083 ceph-mon[29774]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:21.568 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:21 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:21.568 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:21 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:33:21.568 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:21 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:21.568 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:21 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:33:21.568 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:21 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:21.568 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:21 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-07T14:33:21.568 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:21 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-07T14:33:21.568 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:21 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:21.589 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph orch daemon add osd smithi083:vg_nvme/lv_4 2024-04-07T14:33:22.113 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:33:21 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:33:21.859+0000 7f7425309000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-07T14:33:22.113 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:33:21 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:33:21.948+0000 7f7425309000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-07T14:33:22.114 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:33:22 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:33:22.048+0000 7f7425309000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-07T14:33:22.423 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:33:22 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:33:22.203+0000 7f7425309000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-07T14:33:22.423 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:33:22 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:33:22.300+0000 7f7425309000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-07T14:33:22.423 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:33:22 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:33:22.422+0000 7f7425309000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-07T14:33:22.424 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:22 smithi189 ceph-mon[25797]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:33:22.424 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:22 smithi189 ceph-mon[25797]: Reconfiguring mgr.y (unknown last config time)... 2024-04-07T14:33:22.424 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:22 smithi189 ceph-mon[25797]: Reconfiguring daemon mgr.y on smithi083 2024-04-07T14:33:22.424 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:22 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:33:22.424 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:22 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:33:22.424 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:22 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:22.502 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:22 smithi083 ceph-mon[24810]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:33:22.502 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:22 smithi083 ceph-mon[24810]: Reconfiguring mgr.y (unknown last config time)... 2024-04-07T14:33:22.502 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:22 smithi083 ceph-mon[24810]: Reconfiguring daemon mgr.y on smithi083 2024-04-07T14:33:22.503 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:22 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:33:22.503 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:22 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:33:22.503 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:22 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:22.503 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:22 smithi083 ceph-mon[29774]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:33:22.503 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:22 smithi083 ceph-mon[29774]: Reconfiguring mgr.y (unknown last config time)... 2024-04-07T14:33:22.503 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:22 smithi083 ceph-mon[29774]: Reconfiguring daemon mgr.y on smithi083 2024-04-07T14:33:22.503 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:22 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:33:22.503 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:22 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:33:22.503 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:22 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:22.692 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:33:22 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:33:22.518+0000 7f7425309000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-07T14:33:22.692 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:33:22 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:33:22.691+0000 7f7425309000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-07T14:33:22.958 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:33:22 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:33:22.783+0000 7f7425309000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-07T14:33:23.471 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:23 smithi083 ceph-mon[24810]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:23.471 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:23 smithi083 ceph-mon[29774]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:23.630 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:23 smithi189 ceph-mon[25797]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:24.047 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:33:23 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:33:23] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:33:24.047 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:33:23 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:33:23] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:33:24.208 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:33:23 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:33:23.914+0000 7f7425309000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-07T14:33:24.209 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:33:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:33:24.046+0000 7f7425309000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-07T14:33:24.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:24 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:24.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:24 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:33:24.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:24 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:24.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:24 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:33:24.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:24 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:24.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:24 smithi083 ceph-mon[24810]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:33:24.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:24 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:24.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:24 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:24.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:24 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:24.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:24 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:33:24.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:24 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:24.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:24 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:33:24.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:24 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:24.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:24 smithi083 ceph-mon[24810]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:24.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:24 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:24.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:24 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:33:24.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:24 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:24.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:24 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:33:24.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:24 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:24.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:24 smithi083 ceph-mon[29774]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:33:24.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:24 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:24.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:24 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:24.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:24 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:24.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:24 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:33:24.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:24 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:24.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:24 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:33:24.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:24 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:24.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:24 smithi083 ceph-mon[29774]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:24.708 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:33:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:33:24.233+0000 7f7425309000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-07T14:33:24.708 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:33:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:33:24.321+0000 7f7425309000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-07T14:33:24.708 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:33:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:33:24.457+0000 7f7425309000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-07T14:33:24.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:24 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:24.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:24 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:33:24.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:24 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:24.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:24 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:33:24.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:24 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:24.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:24 smithi189 ceph-mon[25797]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:33:24.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:24 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:24.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:24 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:24.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:24 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:24.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:24 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:33:24.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:24 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:24.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:24 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:33:24.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:24 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:24.710 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:24 smithi189 ceph-mon[25797]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:25.079 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:33:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:33:24.776+0000 7f7425309000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-07T14:33:25.079 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:33:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:33:24.996+0000 7f7425309000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-07T14:33:25.079 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:33:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:33:25.080+0000 7f7425309000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-07T14:33:25.358 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:33:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:33:25.249+0000 7f7425309000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-07T14:33:25.573 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:25 smithi083 ceph-mon[24810]: from='client.24124 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:33:25.573 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:25 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-07T14:33:25.573 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:25 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-07T14:33:25.573 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:25 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:25.573 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:25 smithi083 ceph-mon[29774]: from='client.24124 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:33:25.573 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:25 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-07T14:33:25.573 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:25 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-07T14:33:25.573 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:25 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:25.710 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:25 smithi189 ceph-mon[25797]: from='client.24124 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:33:25.710 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:25 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-07T14:33:25.710 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:25 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-07T14:33:25.710 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:25 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:26.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:26 smithi083 ceph-mon[24810]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:26.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:26 smithi083 ceph-mon[29774]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:26.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:26 smithi189 ceph-mon[25797]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:26.458 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:33:26 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:33:26.172+0000 7f7425309000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-07T14:33:26.958 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:33:26 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:33:26.464+0000 7f7425309000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-07T14:33:26.959 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:33:26 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:33:26.558+0000 7f7425309000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-07T14:33:26.959 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:33:26 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:33:26.709+0000 7f7425309000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-07T14:33:27.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:27 smithi083 ceph-mon[24810]: Standby manager daemon x started 2024-04-07T14:33:27.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:27 smithi083 ceph-mon[24810]: from='mgr.? 172.21.15.189:0/3277978054' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-07T14:33:27.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:27 smithi083 ceph-mon[24810]: from='mgr.? 172.21.15.189:0/3277978054' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-07T14:33:27.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:27 smithi083 ceph-mon[24810]: from='mgr.? 172.21.15.189:0/3277978054' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-07T14:33:27.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:27 smithi083 ceph-mon[24810]: from='mgr.? 172.21.15.189:0/3277978054' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-07T14:33:27.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:27 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/3215507700' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a42cc075-c4af-4500-abae-738cfd1c8d97"}]: dispatch 2024-04-07T14:33:27.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:27 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/3215507700' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a42cc075-c4af-4500-abae-738cfd1c8d97"}]': finished 2024-04-07T14:33:27.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:27 smithi083 ceph-mon[24810]: osdmap e5: 1 total, 0 up, 1 in 2024-04-07T14:33:27.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:27 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-07T14:33:27.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:27 smithi083 ceph-mon[29774]: Standby manager daemon x started 2024-04-07T14:33:27.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:27 smithi083 ceph-mon[29774]: from='mgr.? 172.21.15.189:0/3277978054' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-07T14:33:27.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:27 smithi083 ceph-mon[29774]: from='mgr.? 172.21.15.189:0/3277978054' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-07T14:33:27.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:27 smithi083 ceph-mon[29774]: from='mgr.? 172.21.15.189:0/3277978054' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-07T14:33:27.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:27 smithi083 ceph-mon[29774]: from='mgr.? 172.21.15.189:0/3277978054' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-07T14:33:27.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:27 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/3215507700' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a42cc075-c4af-4500-abae-738cfd1c8d97"}]: dispatch 2024-04-07T14:33:27.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:27 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/3215507700' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a42cc075-c4af-4500-abae-738cfd1c8d97"}]': finished 2024-04-07T14:33:27.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:27 smithi083 ceph-mon[29774]: osdmap e5: 1 total, 0 up, 1 in 2024-04-07T14:33:27.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:27 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-07T14:33:27.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:27 smithi189 ceph-mon[25797]: Standby manager daemon x started 2024-04-07T14:33:27.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:27 smithi189 ceph-mon[25797]: from='mgr.? 172.21.15.189:0/3277978054' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-07T14:33:27.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:27 smithi189 ceph-mon[25797]: from='mgr.? 172.21.15.189:0/3277978054' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-07T14:33:27.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:27 smithi189 ceph-mon[25797]: from='mgr.? 172.21.15.189:0/3277978054' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-07T14:33:27.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:27 smithi189 ceph-mon[25797]: from='mgr.? 172.21.15.189:0/3277978054' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-07T14:33:27.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:27 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/3215507700' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a42cc075-c4af-4500-abae-738cfd1c8d97"}]: dispatch 2024-04-07T14:33:27.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:27 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/3215507700' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a42cc075-c4af-4500-abae-738cfd1c8d97"}]': finished 2024-04-07T14:33:27.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:27 smithi189 ceph-mon[25797]: osdmap e5: 1 total, 0 up, 1 in 2024-04-07T14:33:27.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:27 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-07T14:33:28.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:28 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/1768435469' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-07T14:33:28.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:28 smithi083 ceph-mon[24810]: mgrmap e14: y(active, since 65s), standbys: x 2024-04-07T14:33:28.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:28 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-07T14:33:28.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:28 smithi083 ceph-mon[24810]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:28.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:28 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/1768435469' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-07T14:33:28.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:28 smithi083 ceph-mon[29774]: mgrmap e14: y(active, since 65s), standbys: x 2024-04-07T14:33:28.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:28 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-07T14:33:28.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:28 smithi083 ceph-mon[29774]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:28.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:28 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/1768435469' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-07T14:33:28.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:28 smithi189 ceph-mon[25797]: mgrmap e14: y(active, since 65s), standbys: x 2024-04-07T14:33:28.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:28 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-07T14:33:28.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:28 smithi189 ceph-mon[25797]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:31.214 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:31 smithi083 ceph-mon[24810]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:31.214 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:31 smithi083 ceph-mon[29774]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:31.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:31 smithi189 ceph-mon[25797]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:33.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:33 smithi189 ceph-mon[25797]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:33.468 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:33 smithi083 ceph-mon[24810]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:33.468 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:33 smithi083 ceph-mon[29774]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:34.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:34 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-07T14:33:34.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:34 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:34.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:34 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-07T14:33:34.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:34 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:34.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:34 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-07T14:33:34.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:34 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:35.325 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:35 smithi083 ceph-mon[24810]: Deploying daemon osd.0 on smithi083 2024-04-07T14:33:35.326 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:35 smithi083 ceph-mon[24810]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:35.326 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:35 smithi083 ceph-mon[29774]: Deploying daemon osd.0 on smithi083 2024-04-07T14:33:35.326 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:35 smithi083 ceph-mon[29774]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:35.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:35 smithi189 ceph-mon[25797]: Deploying daemon osd.0 on smithi083 2024-04-07T14:33:35.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:35 smithi189 ceph-mon[25797]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:35.676 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:33:35 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:33:35] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:33:36.640 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:36 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:36.641 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:36 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:36.641 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:36 smithi083 ceph-mon[24810]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:36.641 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:36 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:36.641 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:36 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:36.641 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:36 smithi083 ceph-mon[29774]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:36.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:36 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:36.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:36 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:36.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:36 smithi189 ceph-mon[25797]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:39.373 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:39 smithi083 ceph-mon[24810]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:39.373 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:39 smithi083 ceph-mon[29774]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:39.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:39 smithi189 ceph-mon[25797]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:41.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:41 smithi083 ceph-mon[24810]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:41.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:41 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:41.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:41 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:33:41.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:41 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:41.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:41 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:33:41.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:41 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:41.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:41 smithi083 ceph-mon[29774]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:41.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:41 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:41.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:41 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:33:41.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:41 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:41.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:41 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:33:41.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:41 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:41.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:41 smithi189 ceph-mon[25797]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:41.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:41 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:41.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:41 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:33:41.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:41 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:41.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:41 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:33:41.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:41 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:42.926 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:33:42 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:33:42] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:33:43.279 INFO:teuthology.orchestra.run.smithi083.stdout:Created osd(s) 0 on host 'smithi083' 2024-04-07T14:33:43.291 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:43 smithi083 ceph-mon[24810]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:43.291 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:43 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:43.291 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:43 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:43.291 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:43 smithi083 ceph-mon[29774]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:43.291 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:43 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:43.291 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:43 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:43.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:43 smithi189 ceph-mon[25797]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:43.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:43 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:43.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:43 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:43.881 DEBUG:teuthology.orchestra.run.smithi083:osd.0> sudo journalctl -f -n 0 -u ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@osd.0.service 2024-04-07T14:33:43.885 INFO:tasks.cephadm:Deploying osd.1 on smithi083 with /dev/vg_nvme/lv_3... 2024-04-07T14:33:43.885 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-07T14:33:44.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:44 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:44.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:44 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:33:44.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:44 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:44.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:44 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:33:44.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:44 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:33:44.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:44 smithi189 ceph-mon[25797]: Adjusting osd_memory_target on smithi083 to 15040M 2024-04-07T14:33:44.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:44 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:44.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:44 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:44.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:44 smithi189 ceph-mon[25797]: from='osd.0 [v2:172.21.15.83:6802/159265472,v1:172.21.15.83:6803/159265472]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-07T14:33:44.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:44 smithi189 ceph-mon[25797]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-07T14:33:44.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:44 smithi189 ceph-mon[25797]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:44.520 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:44 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:44.520 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:44 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:33:44.520 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:44 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:44.521 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:44 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:33:44.521 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:44 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:33:44.521 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:44 smithi083 ceph-mon[29774]: Adjusting osd_memory_target on smithi083 to 15040M 2024-04-07T14:33:44.521 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:44 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:44.521 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:44 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:44.521 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:44 smithi083 ceph-mon[29774]: from='osd.0 [v2:172.21.15.83:6802/159265472,v1:172.21.15.83:6803/159265472]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-07T14:33:44.521 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:44 smithi083 ceph-mon[29774]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-07T14:33:44.521 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:44 smithi083 ceph-mon[29774]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:44.521 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:44 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:44.521 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:44 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:33:44.521 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:44 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:44.521 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:44 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:33:44.521 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:44 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:33:44.522 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:44 smithi083 ceph-mon[24810]: Adjusting osd_memory_target on smithi083 to 15040M 2024-04-07T14:33:44.522 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:44 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:44.522 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:44 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:44.522 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:44 smithi083 ceph-mon[24810]: from='osd.0 [v2:172.21.15.83:6802/159265472,v1:172.21.15.83:6803/159265472]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-07T14:33:44.522 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:44 smithi083 ceph-mon[24810]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-07T14:33:44.522 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:44 smithi083 ceph-mon[24810]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:45.560 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:45 smithi083 ceph-mon[24810]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-07T14:33:45.560 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:45 smithi083 ceph-mon[24810]: osdmap e6: 1 total, 0 up, 1 in 2024-04-07T14:33:45.560 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:45 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-07T14:33:45.560 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:45 smithi083 ceph-mon[24810]: from='osd.0 [v2:172.21.15.83:6802/159265472,v1:172.21.15.83:6803/159265472]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-04-07T14:33:45.560 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:45 smithi083 ceph-mon[24810]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-04-07T14:33:45.561 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:45 smithi083 ceph-mon[29774]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-07T14:33:45.561 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:45 smithi083 ceph-mon[29774]: osdmap e6: 1 total, 0 up, 1 in 2024-04-07T14:33:45.561 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:45 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-07T14:33:45.561 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:45 smithi083 ceph-mon[29774]: from='osd.0 [v2:172.21.15.83:6802/159265472,v1:172.21.15.83:6803/159265472]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-04-07T14:33:45.561 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:45 smithi083 ceph-mon[29774]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-04-07T14:33:45.561 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 07 14:33:45 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-0[32973]: 2024-04-07T14:33:45.336+0000 7f914cbed700 -1 osd.0 0 waiting for initial osdmap 2024-04-07T14:33:45.561 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 07 14:33:45 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-0[32973]: 2024-04-07T14:33:45.339+0000 7f9145580700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-07T14:33:45.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:45 smithi189 ceph-mon[25797]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-07T14:33:45.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:45 smithi189 ceph-mon[25797]: osdmap e6: 1 total, 0 up, 1 in 2024-04-07T14:33:45.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:45 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-07T14:33:45.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:45 smithi189 ceph-mon[25797]: from='osd.0 [v2:172.21.15.83:6802/159265472,v1:172.21.15.83:6803/159265472]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-04-07T14:33:45.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:45 smithi189 ceph-mon[25797]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-04-07T14:33:46.054 INFO:teuthology.orchestra.run.smithi083.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-07T14:33:46.054 INFO:teuthology.orchestra.run.smithi083.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-07T14:33:46.054 INFO:teuthology.orchestra.run.smithi083.stderr: stderr: 10+0 records in 2024-04-07T14:33:46.054 INFO:teuthology.orchestra.run.smithi083.stderr:10+0 records out 2024-04-07T14:33:46.054 INFO:teuthology.orchestra.run.smithi083.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.012715 s, 825 MB/s 2024-04-07T14:33:46.054 INFO:teuthology.orchestra.run.smithi083.stderr:--> Zapping successful for: 2024-04-07T14:33:46.639 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:46 smithi083 ceph-mon[24810]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2024-04-07T14:33:46.640 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:46 smithi083 ceph-mon[24810]: osdmap e7: 1 total, 0 up, 1 in 2024-04-07T14:33:46.640 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:46 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-07T14:33:46.640 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:46 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-07T14:33:46.640 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:46 smithi083 ceph-mon[24810]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:46.640 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:46 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-07T14:33:46.640 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:46 smithi083 ceph-mon[29774]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2024-04-07T14:33:46.640 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:46 smithi083 ceph-mon[29774]: osdmap e7: 1 total, 0 up, 1 in 2024-04-07T14:33:46.640 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:46 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-07T14:33:46.640 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:46 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-07T14:33:46.640 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:46 smithi083 ceph-mon[29774]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:46.641 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:46 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-07T14:33:46.664 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph orch daemon add osd smithi083:vg_nvme/lv_3 2024-04-07T14:33:46.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:46 smithi189 ceph-mon[25797]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2024-04-07T14:33:46.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:46 smithi189 ceph-mon[25797]: osdmap e7: 1 total, 0 up, 1 in 2024-04-07T14:33:46.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:46 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-07T14:33:46.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:46 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-07T14:33:46.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:46 smithi189 ceph-mon[25797]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-07T14:33:46.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:46 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-07T14:33:47.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:47 smithi083 ceph-mon[29774]: purged_snaps scrub starts 2024-04-07T14:33:47.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:47 smithi083 ceph-mon[29774]: purged_snaps scrub ok 2024-04-07T14:33:47.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:47 smithi083 ceph-mon[29774]: osd.0 [v2:172.21.15.83:6802/159265472,v1:172.21.15.83:6803/159265472] boot 2024-04-07T14:33:47.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:47 smithi083 ceph-mon[29774]: osdmap e8: 1 total, 1 up, 1 in 2024-04-07T14:33:47.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:47 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-07T14:33:47.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:47 smithi083 ceph-mon[24810]: purged_snaps scrub starts 2024-04-07T14:33:47.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:47 smithi083 ceph-mon[24810]: purged_snaps scrub ok 2024-04-07T14:33:47.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:47 smithi083 ceph-mon[24810]: osd.0 [v2:172.21.15.83:6802/159265472,v1:172.21.15.83:6803/159265472] boot 2024-04-07T14:33:47.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:47 smithi083 ceph-mon[24810]: osdmap e8: 1 total, 1 up, 1 in 2024-04-07T14:33:47.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:47 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-07T14:33:47.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:47 smithi189 ceph-mon[25797]: purged_snaps scrub starts 2024-04-07T14:33:47.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:47 smithi189 ceph-mon[25797]: purged_snaps scrub ok 2024-04-07T14:33:47.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:47 smithi189 ceph-mon[25797]: osd.0 [v2:172.21.15.83:6802/159265472,v1:172.21.15.83:6803/159265472] boot 2024-04-07T14:33:47.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:47 smithi189 ceph-mon[25797]: osdmap e8: 1 total, 1 up, 1 in 2024-04-07T14:33:47.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:47 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-07T14:33:48.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:48 smithi083 ceph-mon[24810]: osdmap e9: 1 total, 1 up, 1 in 2024-04-07T14:33:48.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:48 smithi083 ceph-mon[24810]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-07T14:33:48.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:48 smithi083 ceph-mon[29774]: osdmap e9: 1 total, 1 up, 1 in 2024-04-07T14:33:48.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:48 smithi083 ceph-mon[29774]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-07T14:33:48.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:48 smithi189 ceph-mon[25797]: osdmap e9: 1 total, 1 up, 1 in 2024-04-07T14:33:48.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:48 smithi189 ceph-mon[25797]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-07T14:33:49.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:49 smithi083 ceph-mon[24810]: from='client.24151 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:33:49.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:49 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-07T14:33:49.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:49 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-07T14:33:49.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:49 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:49.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:49 smithi083 ceph-mon[29774]: from='client.24151 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:33:49.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:49 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-07T14:33:49.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:49 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-07T14:33:49.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:49 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:49.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:49 smithi189 ceph-mon[25797]: from='client.24151 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:33:49.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:49 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-07T14:33:49.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:49 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-07T14:33:49.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:49 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:51.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:51 smithi189 ceph-mon[25797]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-07T14:33:51.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:51 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/2457931805' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c0b76e30-ff29-43b7-a8d1-90f9bef15b7e"}]: dispatch 2024-04-07T14:33:51.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:51 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/2457931805' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c0b76e30-ff29-43b7-a8d1-90f9bef15b7e"}]': finished 2024-04-07T14:33:51.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:51 smithi189 ceph-mon[25797]: osdmap e10: 2 total, 1 up, 2 in 2024-04-07T14:33:51.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:51 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-07T14:33:51.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:51 smithi083 ceph-mon[24810]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-07T14:33:51.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:51 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/2457931805' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c0b76e30-ff29-43b7-a8d1-90f9bef15b7e"}]: dispatch 2024-04-07T14:33:51.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:51 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/2457931805' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c0b76e30-ff29-43b7-a8d1-90f9bef15b7e"}]': finished 2024-04-07T14:33:51.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:51 smithi083 ceph-mon[24810]: osdmap e10: 2 total, 1 up, 2 in 2024-04-07T14:33:51.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:51 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-07T14:33:51.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:51 smithi083 ceph-mon[29774]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-07T14:33:51.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:51 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/2457931805' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c0b76e30-ff29-43b7-a8d1-90f9bef15b7e"}]: dispatch 2024-04-07T14:33:51.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:51 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/2457931805' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c0b76e30-ff29-43b7-a8d1-90f9bef15b7e"}]': finished 2024-04-07T14:33:51.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:51 smithi083 ceph-mon[29774]: osdmap e10: 2 total, 1 up, 2 in 2024-04-07T14:33:51.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:51 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-07T14:33:52.093 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:33:51 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:33:51] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:33:53.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:52 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:53.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:52 smithi083 ceph-mon[24810]: Detected new or changed devices on smithi083 2024-04-07T14:33:53.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:52 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:53.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:52 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/1063715842' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-07T14:33:53.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:52 smithi083 ceph-mon[24810]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-07T14:33:53.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:52 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:53.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:52 smithi083 ceph-mon[29774]: Detected new or changed devices on smithi083 2024-04-07T14:33:53.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:52 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:53.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:52 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/1063715842' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-07T14:33:53.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:52 smithi083 ceph-mon[29774]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-07T14:33:53.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:52 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:53.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:52 smithi189 ceph-mon[25797]: Detected new or changed devices on smithi083 2024-04-07T14:33:53.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:52 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:33:53.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:52 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/1063715842' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-07T14:33:53.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:52 smithi189 ceph-mon[25797]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-07T14:33:55.324 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:55 smithi083 ceph-mon[24810]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-07T14:33:55.325 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:55 smithi083 ceph-mon[29774]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-07T14:33:55.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:55 smithi189 ceph-mon[25797]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-07T14:33:57.304 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:57 smithi083 ceph-mon[24810]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-07T14:33:57.304 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:57 smithi083 ceph-mon[29774]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-07T14:33:57.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:57 smithi189 ceph-mon[25797]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-07T14:33:59.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:59 smithi083 ceph-mon[29774]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-07T14:33:59.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:59 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-07T14:33:59.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:33:59 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:59.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:59 smithi083 ceph-mon[24810]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-07T14:33:59.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:59 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-07T14:33:59.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:33:59 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:33:59.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:59 smithi189 ceph-mon[25797]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-07T14:33:59.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:59 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-07T14:33:59.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:33:59 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:00.312 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:00 smithi083 ceph-mon[24810]: Deploying daemon osd.1 on smithi083 2024-04-07T14:34:00.313 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:00 smithi083 ceph-mon[29774]: Deploying daemon osd.1 on smithi083 2024-04-07T14:34:00.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:00 smithi189 ceph-mon[25797]: Deploying daemon osd.1 on smithi083 2024-04-07T14:34:01.379 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:01 smithi083 ceph-mon[24810]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-07T14:34:01.379 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:01 smithi083 ceph-mon[29774]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-07T14:34:01.609 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:01 smithi189 ceph-mon[25797]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-07T14:34:03.028 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:34:02 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:34:02] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:34:03.370 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:03 smithi083 ceph-mon[24810]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-07T14:34:03.370 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:03 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:03.370 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:03 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:03.370 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:03 smithi083 ceph-mon[29774]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-07T14:34:03.371 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:03 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:03.371 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:03 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:03.523 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:03 smithi189 ceph-mon[25797]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-07T14:34:03.523 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:03 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:03.523 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:03 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:04.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:04 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:04.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:04 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:34:04.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:04 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:04.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:04 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:34:04.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:04 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:04.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:04 smithi083 ceph-mon[24810]: pgmap v51: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-07T14:34:04.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:04 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:04.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:04 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:34:04.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:04 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:04.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:04 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:34:04.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:04 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:04.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:04 smithi083 ceph-mon[29774]: pgmap v51: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-07T14:34:04.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:04 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:04.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:04 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:34:04.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:04 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:04.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:04 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:34:04.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:04 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:04.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:04 smithi189 ceph-mon[25797]: pgmap v51: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-07T14:34:06.620 INFO:teuthology.orchestra.run.smithi083.stdout:Created osd(s) 1 on host 'smithi083' 2024-04-07T14:34:07.215 DEBUG:teuthology.orchestra.run.smithi083:osd.1> sudo journalctl -f -n 0 -u ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@osd.1.service 2024-04-07T14:34:07.217 INFO:tasks.cephadm:Deploying osd.2 on smithi083 with /dev/vg_nvme/lv_2... 2024-04-07T14:34:07.218 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-07T14:34:07.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:07 smithi189 ceph-mon[25797]: pgmap v52: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-07T14:34:07.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:07 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:34:07.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:07 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:07.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:07 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:07.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:07 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:34:07.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:07 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:34:07.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:07 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:34:07.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:07 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:07.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:07 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:07.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:07 smithi189 ceph-mon[25797]: from='osd.1 [v2:172.21.15.83:6810/4165704838,v1:172.21.15.83:6811/4165704838]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-07T14:34:07.559 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:07 smithi083 ceph-mon[24810]: pgmap v52: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-07T14:34:07.560 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:07 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:34:07.560 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:07 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:07.560 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:07 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:07.560 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:07 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:34:07.560 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:07 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:34:07.560 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:07 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:34:07.560 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:07 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:07.560 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:07 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:07.560 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:07 smithi083 ceph-mon[24810]: from='osd.1 [v2:172.21.15.83:6810/4165704838,v1:172.21.15.83:6811/4165704838]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-07T14:34:07.560 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:07 smithi083 ceph-mon[29774]: pgmap v52: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-07T14:34:07.560 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:07 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:34:07.560 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:07 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:07.561 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:07 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:07.561 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:07 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:34:07.561 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:07 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:34:07.561 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:07 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:34:07.561 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:07 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:07.561 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:07 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:07.561 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:07 smithi083 ceph-mon[29774]: from='osd.1 [v2:172.21.15.83:6810/4165704838,v1:172.21.15.83:6811/4165704838]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-07T14:34:08.506 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:08 smithi083 ceph-mon[24810]: Adjusting osd_memory_target on smithi083 to 7520M 2024-04-07T14:34:08.507 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:08 smithi083 ceph-mon[24810]: from='osd.1 [v2:172.21.15.83:6810/4165704838,v1:172.21.15.83:6811/4165704838]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-07T14:34:08.507 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:08 smithi083 ceph-mon[24810]: osdmap e11: 2 total, 1 up, 2 in 2024-04-07T14:34:08.507 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:08 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-07T14:34:08.507 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:08 smithi083 ceph-mon[24810]: from='osd.1 [v2:172.21.15.83:6810/4165704838,v1:172.21.15.83:6811/4165704838]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-04-07T14:34:08.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:08 smithi083 ceph-mon[29774]: Adjusting osd_memory_target on smithi083 to 7520M 2024-04-07T14:34:08.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:08 smithi083 ceph-mon[29774]: from='osd.1 [v2:172.21.15.83:6810/4165704838,v1:172.21.15.83:6811/4165704838]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-07T14:34:08.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:08 smithi083 ceph-mon[29774]: osdmap e11: 2 total, 1 up, 2 in 2024-04-07T14:34:08.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:08 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-07T14:34:08.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:08 smithi083 ceph-mon[29774]: from='osd.1 [v2:172.21.15.83:6810/4165704838,v1:172.21.15.83:6811/4165704838]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-04-07T14:34:08.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:08 smithi189 ceph-mon[25797]: Adjusting osd_memory_target on smithi083 to 7520M 2024-04-07T14:34:08.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:08 smithi189 ceph-mon[25797]: from='osd.1 [v2:172.21.15.83:6810/4165704838,v1:172.21.15.83:6811/4165704838]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-07T14:34:08.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:08 smithi189 ceph-mon[25797]: osdmap e11: 2 total, 1 up, 2 in 2024-04-07T14:34:08.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:08 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-07T14:34:08.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:08 smithi189 ceph-mon[25797]: from='osd.1 [v2:172.21.15.83:6810/4165704838,v1:172.21.15.83:6811/4165704838]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-04-07T14:34:08.773 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 07 14:34:08 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-1[35861]: 2024-04-07T14:34:08.663+0000 7fa59ff7b700 -1 osd.1 0 waiting for initial osdmap 2024-04-07T14:34:08.773 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 07 14:34:08 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-1[35861]: 2024-04-07T14:34:08.667+0000 7fa59a912700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-07T14:34:09.244 INFO:teuthology.orchestra.run.smithi083.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-07T14:34:09.244 INFO:teuthology.orchestra.run.smithi083.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-07T14:34:09.244 INFO:teuthology.orchestra.run.smithi083.stderr: stderr: 10+0 records in 2024-04-07T14:34:09.244 INFO:teuthology.orchestra.run.smithi083.stderr:10+0 records out 2024-04-07T14:34:09.244 INFO:teuthology.orchestra.run.smithi083.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0126573 s, 828 MB/s 2024-04-07T14:34:09.244 INFO:teuthology.orchestra.run.smithi083.stderr:--> Zapping successful for: 2024-04-07T14:34:09.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:09 smithi083 ceph-mon[24810]: pgmap v54: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-07T14:34:09.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:09 smithi083 ceph-mon[24810]: from='osd.1 [v2:172.21.15.83:6810/4165704838,v1:172.21.15.83:6811/4165704838]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2024-04-07T14:34:09.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:09 smithi083 ceph-mon[24810]: osdmap e12: 2 total, 1 up, 2 in 2024-04-07T14:34:09.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:09 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-07T14:34:09.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:09 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-07T14:34:09.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:09 smithi083 ceph-mon[29774]: pgmap v54: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-07T14:34:09.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:09 smithi083 ceph-mon[29774]: from='osd.1 [v2:172.21.15.83:6810/4165704838,v1:172.21.15.83:6811/4165704838]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2024-04-07T14:34:09.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:09 smithi083 ceph-mon[29774]: osdmap e12: 2 total, 1 up, 2 in 2024-04-07T14:34:09.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:09 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-07T14:34:09.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:09 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-07T14:34:09.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:09 smithi189 ceph-mon[25797]: pgmap v54: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-07T14:34:09.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:09 smithi189 ceph-mon[25797]: from='osd.1 [v2:172.21.15.83:6810/4165704838,v1:172.21.15.83:6811/4165704838]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2024-04-07T14:34:09.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:09 smithi189 ceph-mon[25797]: osdmap e12: 2 total, 1 up, 2 in 2024-04-07T14:34:09.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:09 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-07T14:34:09.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:09 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-07T14:34:09.886 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph orch daemon add osd smithi083:vg_nvme/lv_2 2024-04-07T14:34:10.466 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:10 smithi083 ceph-mon[24810]: purged_snaps scrub starts 2024-04-07T14:34:10.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:10 smithi083 ceph-mon[24810]: purged_snaps scrub ok 2024-04-07T14:34:10.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:10 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-07T14:34:10.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:10 smithi083 ceph-mon[24810]: osd.1 [v2:172.21.15.83:6810/4165704838,v1:172.21.15.83:6811/4165704838] boot 2024-04-07T14:34:10.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:10 smithi083 ceph-mon[24810]: osdmap e13: 2 total, 2 up, 2 in 2024-04-07T14:34:10.467 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:10 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-07T14:34:10.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:10 smithi083 ceph-mon[29774]: purged_snaps scrub starts 2024-04-07T14:34:10.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:10 smithi083 ceph-mon[29774]: purged_snaps scrub ok 2024-04-07T14:34:10.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:10 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-07T14:34:10.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:10 smithi083 ceph-mon[29774]: osd.1 [v2:172.21.15.83:6810/4165704838,v1:172.21.15.83:6811/4165704838] boot 2024-04-07T14:34:10.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:10 smithi083 ceph-mon[29774]: osdmap e13: 2 total, 2 up, 2 in 2024-04-07T14:34:10.467 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:10 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-07T14:34:10.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:10 smithi189 ceph-mon[25797]: purged_snaps scrub starts 2024-04-07T14:34:10.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:10 smithi189 ceph-mon[25797]: purged_snaps scrub ok 2024-04-07T14:34:10.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:10 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-07T14:34:10.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:10 smithi189 ceph-mon[25797]: osd.1 [v2:172.21.15.83:6810/4165704838,v1:172.21.15.83:6811/4165704838] boot 2024-04-07T14:34:10.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:10 smithi189 ceph-mon[25797]: osdmap e13: 2 total, 2 up, 2 in 2024-04-07T14:34:10.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:10 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-07T14:34:11.506 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:11 smithi083 ceph-mon[24810]: pgmap v57: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-07T14:34:11.507 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:11 smithi083 ceph-mon[24810]: osdmap e14: 2 total, 2 up, 2 in 2024-04-07T14:34:11.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:11 smithi083 ceph-mon[29774]: pgmap v57: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-07T14:34:11.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:11 smithi083 ceph-mon[29774]: osdmap e14: 2 total, 2 up, 2 in 2024-04-07T14:34:11.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:11 smithi189 ceph-mon[25797]: pgmap v57: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-07T14:34:11.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:11 smithi189 ceph-mon[25797]: osdmap e14: 2 total, 2 up, 2 in 2024-04-07T14:34:12.176 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:34:11 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:34:11] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:34:13.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:12 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:13.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:12 smithi083 ceph-mon[24810]: Detected new or changed devices on smithi083 2024-04-07T14:34:13.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:12 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:13.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:12 smithi083 ceph-mon[24810]: from='client.24178 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:34:13.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:12 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-07T14:34:13.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:12 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-07T14:34:13.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:12 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:13.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:12 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:34:13.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:12 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:13.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:12 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:34:13.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:12 smithi083 ceph-mon[24810]: pgmap v59: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:13.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:12 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:13.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:12 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:13.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:12 smithi083 ceph-mon[29774]: Detected new or changed devices on smithi083 2024-04-07T14:34:13.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:12 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:13.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:12 smithi083 ceph-mon[29774]: from='client.24178 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:34:13.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:12 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-07T14:34:13.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:12 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-07T14:34:13.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:12 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:13.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:12 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:34:13.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:12 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:13.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:12 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:34:13.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:12 smithi083 ceph-mon[29774]: pgmap v59: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:13.178 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:12 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:13.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:12 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:13.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:12 smithi189 ceph-mon[25797]: Detected new or changed devices on smithi083 2024-04-07T14:34:13.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:12 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:13.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:12 smithi189 ceph-mon[25797]: from='client.24178 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:34:13.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:12 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-07T14:34:13.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:12 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-07T14:34:13.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:12 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:13.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:12 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:34:13.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:12 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:13.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:12 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:34:13.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:12 smithi189 ceph-mon[25797]: pgmap v59: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:13.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:12 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:14.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:13 smithi083 ceph-mon[24810]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-04-07T14:34:14.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:13 smithi083 ceph-mon[29774]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-04-07T14:34:14.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:13 smithi189 ceph-mon[25797]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-04-07T14:34:15.141 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:14 smithi083 ceph-mon[24810]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:15.141 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:14 smithi083 ceph-mon[29774]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:15.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:14 smithi189 ceph-mon[25797]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:15.426 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:34:15 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:34:15] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:34:16.407 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:16 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:16.407 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:16 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:16.407 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:16 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:34:16.407 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:16 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:16.407 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:16 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:34:16.407 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:16 smithi083 ceph-mon[24810]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:16.407 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:16 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:16.408 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:16 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:16.408 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:16 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:16.408 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:16 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:34:16.408 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:16 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:16.408 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:16 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:34:16.408 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:16 smithi083 ceph-mon[29774]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:16.408 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:16 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:16.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:16 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:16.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:16 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:16.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:16 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:34:16.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:16 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:16.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:16 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:34:16.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:16 smithi189 ceph-mon[25797]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:16.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:16 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:17.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:17 smithi083 ceph-mon[24810]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-04-07T14:34:17.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:17 smithi083 ceph-mon[24810]: Cluster is now healthy 2024-04-07T14:34:17.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:17 smithi083 ceph-mon[29774]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-04-07T14:34:17.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:17 smithi083 ceph-mon[29774]: Cluster is now healthy 2024-04-07T14:34:17.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:17 smithi189 ceph-mon[25797]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-04-07T14:34:17.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:17 smithi189 ceph-mon[25797]: Cluster is now healthy 2024-04-07T14:34:18.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:18 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/4273696972' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fe1ee70e-866b-463e-b7fd-2a2057f44b80"}]: dispatch 2024-04-07T14:34:18.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:18 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/4273696972' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fe1ee70e-866b-463e-b7fd-2a2057f44b80"}]': finished 2024-04-07T14:34:18.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:18 smithi083 ceph-mon[24810]: osdmap e15: 3 total, 2 up, 3 in 2024-04-07T14:34:18.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:18 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-07T14:34:18.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:18 smithi083 ceph-mon[24810]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:18.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:18 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/3068635362' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-07T14:34:18.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:18 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/4273696972' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fe1ee70e-866b-463e-b7fd-2a2057f44b80"}]: dispatch 2024-04-07T14:34:18.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:18 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/4273696972' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fe1ee70e-866b-463e-b7fd-2a2057f44b80"}]': finished 2024-04-07T14:34:18.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:18 smithi083 ceph-mon[29774]: osdmap e15: 3 total, 2 up, 3 in 2024-04-07T14:34:18.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:18 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-07T14:34:18.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:18 smithi083 ceph-mon[29774]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:18.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:18 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/3068635362' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-07T14:34:18.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:18 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/4273696972' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fe1ee70e-866b-463e-b7fd-2a2057f44b80"}]: dispatch 2024-04-07T14:34:18.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:18 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/4273696972' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fe1ee70e-866b-463e-b7fd-2a2057f44b80"}]': finished 2024-04-07T14:34:18.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:18 smithi189 ceph-mon[25797]: osdmap e15: 3 total, 2 up, 3 in 2024-04-07T14:34:18.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:18 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-07T14:34:18.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:18 smithi189 ceph-mon[25797]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:18.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:18 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/3068635362' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-07T14:34:20.340 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:20 smithi083 ceph-mon[24810]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:20.340 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:20 smithi083 ceph-mon[29774]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:20.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:20 smithi189 ceph-mon[25797]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:22.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:22 smithi083 ceph-mon[24810]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:22.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:22 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:34:22.508 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:22 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:34:22.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:22 smithi083 ceph-mon[29774]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:22.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:22 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:34:22.508 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:22 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:34:22.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:22 smithi189 ceph-mon[25797]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:22.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:22 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:34:22.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:22 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:34:23.176 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:34:22 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:34:22] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:34:24.096 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:23 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:24.096 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:23 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:24.097 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:23 smithi189 ceph-mon[25797]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:24.097 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:23 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-07T14:34:24.097 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:23 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:24.290 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:23 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:24.290 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:23 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:24.290 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:23 smithi083 ceph-mon[24810]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:24.291 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:23 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-07T14:34:24.291 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:23 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:24.291 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:23 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:24.291 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:23 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:24.291 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:23 smithi083 ceph-mon[29774]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:24.291 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:23 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-07T14:34:24.291 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:23 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:25.127 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:24 smithi189 ceph-mon[25797]: Deploying daemon osd.2 on smithi083 2024-04-07T14:34:25.183 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:24 smithi083 ceph-mon[24810]: Deploying daemon osd.2 on smithi083 2024-04-07T14:34:25.184 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:24 smithi083 ceph-mon[29774]: Deploying daemon osd.2 on smithi083 2024-04-07T14:34:26.279 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:26 smithi083 ceph-mon[24810]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:26.279 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:26 smithi083 ceph-mon[29774]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:26.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:26 smithi189 ceph-mon[25797]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:28.595 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:28 smithi083 ceph-mon[24810]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:28.595 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:28 smithi083 ceph-mon[29774]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:28.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:28 smithi189 ceph-mon[25797]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:30.010 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:29 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:30.010 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:29 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:34:30.010 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:29 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:30.010 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:29 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:34:30.010 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:29 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:30.010 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:29 smithi083 ceph-mon[24810]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:30.011 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:29 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:30.011 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:29 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:34:30.011 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:29 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:30.011 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:29 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:34:30.011 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:29 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:30.011 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:29 smithi083 ceph-mon[29774]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:30.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:29 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:30.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:29 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:34:30.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:29 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:30.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:29 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:34:30.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:29 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:30.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:29 smithi189 ceph-mon[25797]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:31.397 INFO:teuthology.orchestra.run.smithi083.stdout:Created osd(s) 2 on host 'smithi083' 2024-04-07T14:34:32.195 DEBUG:teuthology.orchestra.run.smithi083:osd.2> sudo journalctl -f -n 0 -u ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@osd.2.service 2024-04-07T14:34:32.198 INFO:tasks.cephadm:Deploying osd.3 on smithi083 with /dev/vg_nvme/lv_1... 2024-04-07T14:34:32.198 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-07T14:34:32.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:32 smithi083 ceph-mon[29774]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:32.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:32 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:32.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:32 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:34:32.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:32 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:32.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:32 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:34:32.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:32 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:34:32.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:32 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:34:32.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:32 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:34:32.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:32 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:32.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:32 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:32.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:32 smithi083 ceph-mon[29774]: from='osd.2 [v2:172.21.15.83:6818/718005878,v1:172.21.15.83:6819/718005878]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-07T14:34:32.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:32 smithi083 ceph-mon[24810]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:32.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:32 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:32.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:32 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:34:32.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:32 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:32.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:32 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:34:32.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:32 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:34:32.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:32 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:34:32.428 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:32 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:34:32.428 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:32 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:32.428 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:32 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:32.428 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:32 smithi083 ceph-mon[24810]: from='osd.2 [v2:172.21.15.83:6818/718005878,v1:172.21.15.83:6819/718005878]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-07T14:34:32.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:32 smithi189 ceph-mon[25797]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:32.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:32 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:32.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:32 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:34:32.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:32 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:32.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:32 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:34:32.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:32 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:34:32.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:32 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:34:32.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:32 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:34:32.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:32 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:32.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:32 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:32.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:32 smithi189 ceph-mon[25797]: from='osd.2 [v2:172.21.15.83:6818/718005878,v1:172.21.15.83:6819/718005878]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-07T14:34:33.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:33 smithi083 ceph-mon[24810]: Adjusting osd_memory_target on smithi083 to 5013M 2024-04-07T14:34:33.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:33 smithi083 ceph-mon[24810]: from='osd.2 [v2:172.21.15.83:6818/718005878,v1:172.21.15.83:6819/718005878]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-07T14:34:33.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:33 smithi083 ceph-mon[24810]: osdmap e16: 3 total, 2 up, 3 in 2024-04-07T14:34:33.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:33 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-07T14:34:33.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:33 smithi083 ceph-mon[24810]: from='osd.2 [v2:172.21.15.83:6818/718005878,v1:172.21.15.83:6819/718005878]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-04-07T14:34:33.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:33 smithi083 ceph-mon[29774]: Adjusting osd_memory_target on smithi083 to 5013M 2024-04-07T14:34:33.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:33 smithi083 ceph-mon[29774]: from='osd.2 [v2:172.21.15.83:6818/718005878,v1:172.21.15.83:6819/718005878]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-07T14:34:33.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:33 smithi083 ceph-mon[29774]: osdmap e16: 3 total, 2 up, 3 in 2024-04-07T14:34:33.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:33 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-07T14:34:33.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:33 smithi083 ceph-mon[29774]: from='osd.2 [v2:172.21.15.83:6818/718005878,v1:172.21.15.83:6819/718005878]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-04-07T14:34:33.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:33 smithi189 ceph-mon[25797]: Adjusting osd_memory_target on smithi083 to 5013M 2024-04-07T14:34:33.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:33 smithi189 ceph-mon[25797]: from='osd.2 [v2:172.21.15.83:6818/718005878,v1:172.21.15.83:6819/718005878]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-07T14:34:33.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:33 smithi189 ceph-mon[25797]: osdmap e16: 3 total, 2 up, 3 in 2024-04-07T14:34:33.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:33 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-07T14:34:33.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:33 smithi189 ceph-mon[25797]: from='osd.2 [v2:172.21.15.83:6818/718005878,v1:172.21.15.83:6819/718005878]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-04-07T14:34:33.694 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 07 14:34:33 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-2[38815]: 2024-04-07T14:34:33.439+0000 7f3059de3700 -1 osd.2 0 waiting for initial osdmap 2024-04-07T14:34:33.694 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 07 14:34:33 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-2[38815]: 2024-04-07T14:34:33.443+0000 7f3053778700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-07T14:34:34.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:34 smithi083 ceph-mon[29774]: pgmap v72: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:34.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:34 smithi083 ceph-mon[29774]: from='osd.2 [v2:172.21.15.83:6818/718005878,v1:172.21.15.83:6819/718005878]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2024-04-07T14:34:34.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:34 smithi083 ceph-mon[29774]: osdmap e17: 3 total, 2 up, 3 in 2024-04-07T14:34:34.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:34 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-07T14:34:34.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:34 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-07T14:34:34.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:34 smithi083 ceph-mon[24810]: pgmap v72: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:34.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:34 smithi083 ceph-mon[24810]: from='osd.2 [v2:172.21.15.83:6818/718005878,v1:172.21.15.83:6819/718005878]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2024-04-07T14:34:34.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:34 smithi083 ceph-mon[24810]: osdmap e17: 3 total, 2 up, 3 in 2024-04-07T14:34:34.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:34 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-07T14:34:34.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:34 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-07T14:34:34.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:34 smithi189 ceph-mon[25797]: pgmap v72: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:34.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:34 smithi189 ceph-mon[25797]: from='osd.2 [v2:172.21.15.83:6818/718005878,v1:172.21.15.83:6819/718005878]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2024-04-07T14:34:34.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:34 smithi189 ceph-mon[25797]: osdmap e17: 3 total, 2 up, 3 in 2024-04-07T14:34:34.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:34 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-07T14:34:34.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:34 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-07T14:34:35.134 INFO:teuthology.orchestra.run.smithi083.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-07T14:34:35.134 INFO:teuthology.orchestra.run.smithi083.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-07T14:34:35.134 INFO:teuthology.orchestra.run.smithi083.stderr: stderr: 10+0 records in 2024-04-07T14:34:35.134 INFO:teuthology.orchestra.run.smithi083.stderr:10+0 records out 2024-04-07T14:34:35.134 INFO:teuthology.orchestra.run.smithi083.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0154526 s, 679 MB/s 2024-04-07T14:34:35.134 INFO:teuthology.orchestra.run.smithi083.stderr:--> Zapping successful for: 2024-04-07T14:34:35.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:35 smithi083 ceph-mon[24810]: purged_snaps scrub starts 2024-04-07T14:34:35.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:35 smithi083 ceph-mon[24810]: purged_snaps scrub ok 2024-04-07T14:34:35.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:35 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-07T14:34:35.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:35 smithi083 ceph-mon[24810]: osd.2 [v2:172.21.15.83:6818/718005878,v1:172.21.15.83:6819/718005878] boot 2024-04-07T14:34:35.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:35 smithi083 ceph-mon[24810]: osdmap e18: 3 total, 3 up, 3 in 2024-04-07T14:34:35.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:35 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-07T14:34:35.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:35 smithi083 ceph-mon[24810]: pgmap v75: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:35.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:35 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:35.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:35 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:35.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:35 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:34:35.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:35 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:35.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:35 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:34:35.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:35 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:35.677 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:34:35 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:34:35] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:34:35.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:35 smithi083 ceph-mon[29774]: purged_snaps scrub starts 2024-04-07T14:34:35.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:35 smithi083 ceph-mon[29774]: purged_snaps scrub ok 2024-04-07T14:34:35.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:35 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-07T14:34:35.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:35 smithi083 ceph-mon[29774]: osd.2 [v2:172.21.15.83:6818/718005878,v1:172.21.15.83:6819/718005878] boot 2024-04-07T14:34:35.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:35 smithi083 ceph-mon[29774]: osdmap e18: 3 total, 3 up, 3 in 2024-04-07T14:34:35.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:35 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-07T14:34:35.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:35 smithi083 ceph-mon[29774]: pgmap v75: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:35.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:35 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:35.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:35 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:35.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:35 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:34:35.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:35 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:35.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:35 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:34:35.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:35 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:35.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:35 smithi189 ceph-mon[25797]: purged_snaps scrub starts 2024-04-07T14:34:35.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:35 smithi189 ceph-mon[25797]: purged_snaps scrub ok 2024-04-07T14:34:35.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:35 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-07T14:34:35.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:35 smithi189 ceph-mon[25797]: osd.2 [v2:172.21.15.83:6818/718005878,v1:172.21.15.83:6819/718005878] boot 2024-04-07T14:34:35.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:35 smithi189 ceph-mon[25797]: osdmap e18: 3 total, 3 up, 3 in 2024-04-07T14:34:35.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:35 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-07T14:34:35.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:35 smithi189 ceph-mon[25797]: pgmap v75: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:35.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:35 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:35.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:35 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:35.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:35 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:34:35.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:35 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:35.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:35 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:34:35.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:35 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:35.834 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph orch daemon add osd smithi083:vg_nvme/lv_1 2024-04-07T14:34:36.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:36 smithi083 ceph-mon[24810]: pgmap v76: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:36.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:36 smithi083 ceph-mon[24810]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-04-07T14:34:36.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:36 smithi083 ceph-mon[24810]: osdmap e19: 3 total, 3 up, 3 in 2024-04-07T14:34:36.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:36 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-07T14:34:36.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:36 smithi083 ceph-mon[29774]: pgmap v76: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:36.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:36 smithi083 ceph-mon[29774]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-04-07T14:34:36.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:36 smithi083 ceph-mon[29774]: osdmap e19: 3 total, 3 up, 3 in 2024-04-07T14:34:36.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:36 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-07T14:34:36.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:36 smithi189 ceph-mon[25797]: pgmap v76: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-07T14:34:36.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:36 smithi189 ceph-mon[25797]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-04-07T14:34:36.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:36 smithi189 ceph-mon[25797]: osdmap e19: 3 total, 3 up, 3 in 2024-04-07T14:34:36.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:36 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-07T14:34:37.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:37 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-07T14:34:37.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:37 smithi083 ceph-mon[24810]: osdmap e20: 3 total, 3 up, 3 in 2024-04-07T14:34:37.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:37 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-07T14:34:37.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:37 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-07T14:34:37.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:37 smithi083 ceph-mon[29774]: osdmap e20: 3 total, 3 up, 3 in 2024-04-07T14:34:37.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:37 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-07T14:34:37.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:37 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-07T14:34:37.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:37 smithi189 ceph-mon[25797]: osdmap e20: 3 total, 3 up, 3 in 2024-04-07T14:34:37.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:37 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-07T14:34:38.043 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 07 14:34:37 smithi083 sudo[40015]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-07T14:34:38.043 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 07 14:34:37 smithi083 sudo[40015]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-07T14:34:38.043 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 07 14:34:37 smithi083 sudo[40015]: pam_unix(sudo:session): session closed for user root 2024-04-07T14:34:38.318 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 07 14:34:38 smithi083 sudo[40019]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-07T14:34:38.318 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 07 14:34:38 smithi083 sudo[40019]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-07T14:34:38.318 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 07 14:34:38 smithi083 sudo[40019]: pam_unix(sudo:session): session closed for user root 2024-04-07T14:34:38.585 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:38 smithi083 ceph-mon[24810]: pgmap v79: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-07T14:34:38.585 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:38 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-07T14:34:38.585 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:38 smithi083 ceph-mon[24810]: osdmap e21: 3 total, 3 up, 3 in 2024-04-07T14:34:38.585 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:38 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:38.585 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:38 smithi083 ceph-mon[29774]: pgmap v79: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-07T14:34:38.585 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:38 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-07T14:34:38.585 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:38 smithi083 ceph-mon[29774]: osdmap e21: 3 total, 3 up, 3 in 2024-04-07T14:34:38.585 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:38 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:38.586 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 07 14:34:38 smithi083 sudo[40040]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-07T14:34:38.586 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 07 14:34:38 smithi083 sudo[40040]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-07T14:34:38.586 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 07 14:34:38 smithi083 sudo[40040]: pam_unix(sudo:session): session closed for user root 2024-04-07T14:34:38.586 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 07 14:34:38 smithi083 sudo[40050]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-07T14:34:38.841 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 07 14:34:38 smithi083 sudo[40050]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-07T14:34:38.841 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 07 14:34:38 smithi083 sudo[40050]: pam_unix(sudo:session): session closed for user root 2024-04-07T14:34:38.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:38 smithi189 ceph-mon[25797]: pgmap v79: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-07T14:34:38.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:38 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-07T14:34:38.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:38 smithi189 ceph-mon[25797]: osdmap e21: 3 total, 3 up, 3 in 2024-04-07T14:34:38.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:38 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:39.119 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 07 14:34:38 smithi083 sudo[40093]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-07T14:34:39.119 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 07 14:34:38 smithi083 sudo[40093]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-07T14:34:39.119 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 07 14:34:38 smithi083 sudo[40093]: pam_unix(sudo:session): session closed for user root 2024-04-07T14:34:39.119 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 07 14:34:39 smithi083 sudo[40157]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-07T14:34:39.370 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 07 14:34:39 smithi083 sudo[40157]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-07T14:34:39.370 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 07 14:34:39 smithi083 sudo[40157]: pam_unix(sudo:session): session closed for user root 2024-04-07T14:34:39.627 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:39 smithi083 sudo[40166]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-07T14:34:39.627 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:39 smithi083 sudo[40166]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-07T14:34:39.627 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:39 smithi083 sudo[40166]: pam_unix(sudo:session): session closed for user root 2024-04-07T14:34:39.881 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:39 smithi083 ceph-mon[29774]: osdmap e22: 3 total, 3 up, 3 in 2024-04-07T14:34:39.881 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:39 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-07T14:34:39.881 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:39 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-07T14:34:39.881 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:39 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:39.881 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:39 smithi083 sudo[40182]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-07T14:34:39.881 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:39 smithi083 sudo[40182]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-07T14:34:39.881 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:39 smithi083 sudo[40182]: pam_unix(sudo:session): session closed for user root 2024-04-07T14:34:39.881 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:39 smithi083 ceph-mon[24810]: osdmap e22: 3 total, 3 up, 3 in 2024-04-07T14:34:39.881 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:39 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-07T14:34:39.881 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:39 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-07T14:34:39.882 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:39 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:39.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:39 smithi189 ceph-mon[25797]: osdmap e22: 3 total, 3 up, 3 in 2024-04-07T14:34:39.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:39 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-07T14:34:39.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:39 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-07T14:34:39.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:39 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:40.137 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:39 smithi083 sudo[40188]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-07T14:34:40.137 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:39 smithi083 sudo[40188]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-07T14:34:40.137 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:39 smithi083 sudo[40188]: pam_unix(sudo:session): session closed for user root 2024-04-07T14:34:40.414 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:40 smithi083 sudo[40191]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-07T14:34:40.414 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:40 smithi083 sudo[40191]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-07T14:34:40.414 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:40 smithi083 sudo[40191]: pam_unix(sudo:session): session closed for user root 2024-04-07T14:34:40.629 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:40 smithi189 sudo[28322]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-07T14:34:40.629 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:40 smithi189 sudo[28322]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-07T14:34:40.629 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:40 smithi189 sudo[28322]: pam_unix(sudo:session): session closed for user root 2024-04-07T14:34:40.629 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:40 smithi189 sudo[28325]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-07T14:34:40.629 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:40 smithi189 sudo[28325]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-07T14:34:40.629 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:40 smithi189 sudo[28325]: pam_unix(sudo:session): session closed for user root 2024-04-07T14:34:40.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:40 smithi083 ceph-mon[24810]: from='client.14307 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:34:40.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:40 smithi083 ceph-mon[24810]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-07T14:34:40.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:40 smithi083 ceph-mon[24810]: pgmap v82: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-07T14:34:40.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:40 smithi083 ceph-mon[24810]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-07T14:34:40.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:40 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-07T14:34:40.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:40 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-07T14:34:40.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:40 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-07T14:34:40.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:40 smithi083 ceph-mon[24810]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-07T14:34:40.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:40 smithi083 ceph-mon[24810]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-07T14:34:40.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:40 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-07T14:34:40.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:40 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-07T14:34:40.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:40 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-07T14:34:40.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:40 smithi083 ceph-mon[24810]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-07T14:34:40.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:40 smithi083 ceph-mon[24810]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-07T14:34:40.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:40 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-07T14:34:40.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:40 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-07T14:34:40.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:40 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-07T14:34:40.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:40 smithi083 ceph-mon[29774]: from='client.14307 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:34:40.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:40 smithi083 ceph-mon[29774]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-07T14:34:40.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:40 smithi083 ceph-mon[29774]: pgmap v82: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-07T14:34:40.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:40 smithi083 ceph-mon[29774]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-07T14:34:40.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:40 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-07T14:34:40.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:40 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-07T14:34:40.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:40 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-07T14:34:40.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:40 smithi083 ceph-mon[29774]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-07T14:34:40.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:40 smithi083 ceph-mon[29774]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-07T14:34:40.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:40 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-07T14:34:40.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:40 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-07T14:34:40.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:40 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-07T14:34:40.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:40 smithi083 ceph-mon[29774]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-07T14:34:40.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:40 smithi083 ceph-mon[29774]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-07T14:34:40.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:40 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-07T14:34:40.679 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:40 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-07T14:34:40.679 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:40 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-07T14:34:40.951 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:40 smithi189 ceph-mon[25797]: from='client.14307 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi083:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:34:40.951 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:40 smithi189 ceph-mon[25797]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-07T14:34:40.951 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:40 smithi189 ceph-mon[25797]: pgmap v82: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-07T14:34:40.951 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:40 smithi189 ceph-mon[25797]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-07T14:34:40.951 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:40 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-07T14:34:40.951 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:40 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-07T14:34:40.951 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:40 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-07T14:34:40.951 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:40 smithi189 ceph-mon[25797]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-07T14:34:40.951 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:40 smithi189 ceph-mon[25797]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-07T14:34:40.952 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:40 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-07T14:34:40.952 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:40 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-07T14:34:40.952 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:40 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-07T14:34:40.952 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:40 smithi189 ceph-mon[25797]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-07T14:34:40.952 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:40 smithi189 ceph-mon[25797]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-07T14:34:40.952 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:40 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-07T14:34:40.952 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:40 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-07T14:34:40.952 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:40 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-07T14:34:41.176 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:34:40 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:34:40] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:34:41.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:41 smithi083 ceph-mon[29774]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-07T14:34:41.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:41 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:41.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:41 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:41.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:41 smithi083 ceph-mon[29774]: Detected new or changed devices on smithi083 2024-04-07T14:34:41.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:41 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:41.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:41 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:34:41.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:41 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:41.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:41 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:34:41.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:41 smithi083 ceph-mon[29774]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-04-07T14:34:41.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:41 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:41.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:41 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/149142843' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "14bfed5e-6a77-4448-855b-258e622311ee"}]: dispatch 2024-04-07T14:34:41.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:41 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/149142843' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "14bfed5e-6a77-4448-855b-258e622311ee"}]': finished 2024-04-07T14:34:41.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:41 smithi083 ceph-mon[29774]: osdmap e23: 4 total, 3 up, 4 in 2024-04-07T14:34:41.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:41 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-07T14:34:41.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:41 smithi083 ceph-mon[24810]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-07T14:34:41.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:41 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:41.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:41 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:41.928 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:41 smithi083 ceph-mon[24810]: Detected new or changed devices on smithi083 2024-04-07T14:34:41.928 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:41 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:41.928 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:41 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:34:41.928 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:41 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:41.928 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:41 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:34:41.928 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:41 smithi083 ceph-mon[24810]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-04-07T14:34:41.928 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:41 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:41.928 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:41 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/149142843' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "14bfed5e-6a77-4448-855b-258e622311ee"}]: dispatch 2024-04-07T14:34:41.928 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:41 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/149142843' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "14bfed5e-6a77-4448-855b-258e622311ee"}]': finished 2024-04-07T14:34:41.928 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:41 smithi083 ceph-mon[24810]: osdmap e23: 4 total, 3 up, 4 in 2024-04-07T14:34:41.928 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:41 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-07T14:34:41.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:41 smithi189 ceph-mon[25797]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-07T14:34:41.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:41 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:41.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:41 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:41.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:41 smithi189 ceph-mon[25797]: Detected new or changed devices on smithi083 2024-04-07T14:34:41.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:41 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:41.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:41 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:34:41.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:41 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:41.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:41 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:34:41.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:41 smithi189 ceph-mon[25797]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-04-07T14:34:41.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:41 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:41.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:41 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/149142843' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "14bfed5e-6a77-4448-855b-258e622311ee"}]: dispatch 2024-04-07T14:34:41.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:41 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/149142843' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "14bfed5e-6a77-4448-855b-258e622311ee"}]': finished 2024-04-07T14:34:41.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:41 smithi189 ceph-mon[25797]: osdmap e23: 4 total, 3 up, 4 in 2024-04-07T14:34:41.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:41 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-07T14:34:42.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:42 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/1575351869' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-07T14:34:42.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:42 smithi083 ceph-mon[24810]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-04-07T14:34:42.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:42 smithi083 ceph-mon[24810]: Cluster is now healthy 2024-04-07T14:34:42.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:42 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/1575351869' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-07T14:34:42.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:42 smithi083 ceph-mon[29774]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-04-07T14:34:42.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:42 smithi083 ceph-mon[29774]: Cluster is now healthy 2024-04-07T14:34:42.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:42 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/1575351869' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-07T14:34:42.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:42 smithi189 ceph-mon[25797]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-04-07T14:34:42.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:42 smithi189 ceph-mon[25797]: Cluster is now healthy 2024-04-07T14:34:43.426 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:34:43 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:34:43] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:34:44.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:44 smithi189 ceph-mon[25797]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-07T14:34:44.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:44 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:44.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:44 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:44.355 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:44 smithi083 ceph-mon[24810]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-07T14:34:44.355 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:44 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:44.355 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:44 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:44.355 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:44 smithi083 ceph-mon[29774]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-07T14:34:44.355 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:44 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:44.355 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:44 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:46.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:46 smithi083 ceph-mon[24810]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-07T14:34:46.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:46 smithi083 ceph-mon[29774]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-07T14:34:46.550 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:46 smithi189 ceph-mon[25797]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-07T14:34:48.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:48 smithi083 ceph-mon[24810]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 341 B/s rd, 32 KiB/s wr, 1 op/s 2024-04-07T14:34:48.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:48 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-07T14:34:48.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:48 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:48.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:48 smithi083 ceph-mon[29774]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 341 B/s rd, 32 KiB/s wr, 1 op/s 2024-04-07T14:34:48.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:48 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-07T14:34:48.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:48 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:48.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:48 smithi189 ceph-mon[25797]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 341 B/s rd, 32 KiB/s wr, 1 op/s 2024-04-07T14:34:48.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:48 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-07T14:34:48.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:48 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:49.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:49 smithi083 ceph-mon[24810]: Deploying daemon osd.3 on smithi083 2024-04-07T14:34:49.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:49 smithi083 ceph-mon[29774]: Deploying daemon osd.3 on smithi083 2024-04-07T14:34:49.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:49 smithi189 ceph-mon[25797]: Deploying daemon osd.3 on smithi083 2024-04-07T14:34:50.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:50 smithi083 ceph-mon[24810]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 255 B/s rd, 24 KiB/s wr, 1 op/s 2024-04-07T14:34:50.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:50 smithi083 ceph-mon[29774]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 255 B/s rd, 24 KiB/s wr, 1 op/s 2024-04-07T14:34:50.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:50 smithi189 ceph-mon[25797]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 255 B/s rd, 24 KiB/s wr, 1 op/s 2024-04-07T14:34:52.336 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:52 smithi083 ceph-mon[24810]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 204 B/s rd, 19 KiB/s wr, 0 op/s 2024-04-07T14:34:52.336 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:52 smithi083 ceph-mon[29774]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 204 B/s rd, 19 KiB/s wr, 0 op/s 2024-04-07T14:34:52.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:52 smithi189 ceph-mon[25797]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 204 B/s rd, 19 KiB/s wr, 0 op/s 2024-04-07T14:34:53.910 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:53 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:53.910 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:53 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:34:53.910 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:53 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:53.910 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:53 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:34:53.910 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:53 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:53.910 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:53 smithi083 ceph-mon[24810]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 175 B/s rd, 17 KiB/s wr, 0 op/s 2024-04-07T14:34:53.910 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:53 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:53.910 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:53 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:34:53.910 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:53 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:53.911 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:53 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:34:53.911 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:53 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:53.911 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:53 smithi083 ceph-mon[29774]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 175 B/s rd, 17 KiB/s wr, 0 op/s 2024-04-07T14:34:53.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:53 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:53.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:53 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:34:53.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:53 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:53.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:53 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:34:53.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:53 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:53.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:53 smithi189 ceph-mon[25797]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 175 B/s rd, 17 KiB/s wr, 0 op/s 2024-04-07T14:34:55.427 INFO:teuthology.orchestra.run.smithi083.stdout:Created osd(s) 3 on host 'smithi083' 2024-04-07T14:34:56.111 DEBUG:teuthology.orchestra.run.smithi083:osd.3> sudo journalctl -f -n 0 -u ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@osd.3.service 2024-04-07T14:34:56.113 INFO:tasks.cephadm:Deploying osd.4 on smithi189 with /dev/vg_nvme/lv_4... 2024-04-07T14:34:56.114 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-07T14:34:56.343 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:56 smithi083 ceph-mon[24810]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 170 B/s rd, 16 KiB/s wr, 0 op/s 2024-04-07T14:34:56.343 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:56 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:34:56.343 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:56 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:56.343 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:56 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:56.343 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:56 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:34:56.343 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:56 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:34:56.344 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:56 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:34:56.344 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:56 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:34:56.344 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:56 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:34:56.344 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:56 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:56.344 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:56 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:56.344 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:56 smithi083 ceph-mon[24810]: from='osd.3 [v2:172.21.15.83:6826/3079630945,v1:172.21.15.83:6827/3079630945]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-07T14:34:56.344 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:56 smithi083 ceph-mon[29774]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 170 B/s rd, 16 KiB/s wr, 0 op/s 2024-04-07T14:34:56.344 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:56 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:34:56.344 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:56 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:56.344 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:56 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:56.344 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:56 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:34:56.345 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:56 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:34:56.345 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:56 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:34:56.345 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:56 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:34:56.345 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:56 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:34:56.345 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:56 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:56.345 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:56 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:56.345 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:56 smithi083 ceph-mon[29774]: from='osd.3 [v2:172.21.15.83:6826/3079630945,v1:172.21.15.83:6827/3079630945]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-07T14:34:56.433 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:56 smithi189 ceph-mon[25797]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 170 B/s rd, 16 KiB/s wr, 0 op/s 2024-04-07T14:34:56.433 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:56 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:34:56.433 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:56 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:34:56.433 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:56 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:56.433 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:56 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:34:56.433 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:56 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:34:56.433 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:56 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:34:56.433 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:56 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:34:56.433 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:56 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:34:56.433 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:56 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:56.433 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:56 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:34:56.434 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:56 smithi189 ceph-mon[25797]: from='osd.3 [v2:172.21.15.83:6826/3079630945,v1:172.21.15.83:6827/3079630945]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-07T14:34:57.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:57 smithi083 ceph-mon[29774]: Adjusting osd_memory_target on smithi083 to 3760M 2024-04-07T14:34:57.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:57 smithi083 ceph-mon[29774]: from='osd.3 [v2:172.21.15.83:6826/3079630945,v1:172.21.15.83:6827/3079630945]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-07T14:34:57.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:57 smithi083 ceph-mon[29774]: osdmap e24: 4 total, 3 up, 4 in 2024-04-07T14:34:57.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:57 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-07T14:34:57.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:57 smithi083 ceph-mon[29774]: from='osd.3 [v2:172.21.15.83:6826/3079630945,v1:172.21.15.83:6827/3079630945]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-04-07T14:34:57.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:57 smithi083 ceph-mon[24810]: Adjusting osd_memory_target on smithi083 to 3760M 2024-04-07T14:34:57.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:57 smithi083 ceph-mon[24810]: from='osd.3 [v2:172.21.15.83:6826/3079630945,v1:172.21.15.83:6827/3079630945]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-07T14:34:57.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:57 smithi083 ceph-mon[24810]: osdmap e24: 4 total, 3 up, 4 in 2024-04-07T14:34:57.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:57 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-07T14:34:57.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:57 smithi083 ceph-mon[24810]: from='osd.3 [v2:172.21.15.83:6826/3079630945,v1:172.21.15.83:6827/3079630945]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-04-07T14:34:57.521 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:57 smithi189 ceph-mon[25797]: Adjusting osd_memory_target on smithi083 to 3760M 2024-04-07T14:34:57.521 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:57 smithi189 ceph-mon[25797]: from='osd.3 [v2:172.21.15.83:6826/3079630945,v1:172.21.15.83:6827/3079630945]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-07T14:34:57.521 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:57 smithi189 ceph-mon[25797]: osdmap e24: 4 total, 3 up, 4 in 2024-04-07T14:34:57.522 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:57 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-07T14:34:57.522 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:57 smithi189 ceph-mon[25797]: from='osd.3 [v2:172.21.15.83:6826/3079630945,v1:172.21.15.83:6827/3079630945]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]: dispatch 2024-04-07T14:34:57.785 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 07 14:34:57 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-3[41825]: 2024-04-07T14:34:57.448+0000 7fe8f9532700 -1 osd.3 0 waiting for initial osdmap 2024-04-07T14:34:57.785 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 07 14:34:57 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-3[41825]: 2024-04-07T14:34:57.453+0000 7fe8f4ecb700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-07T14:34:57.993 INFO:teuthology.orchestra.run.smithi189.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-07T14:34:57.993 INFO:teuthology.orchestra.run.smithi189.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-07T14:34:57.993 INFO:teuthology.orchestra.run.smithi189.stderr: stderr: 10+0 records in 2024-04-07T14:34:57.993 INFO:teuthology.orchestra.run.smithi189.stderr:10+0 records out 2024-04-07T14:34:57.993 INFO:teuthology.orchestra.run.smithi189.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0112705 s, 930 MB/s 2024-04-07T14:34:57.993 INFO:teuthology.orchestra.run.smithi189.stderr:--> Zapping successful for: 2024-04-07T14:34:58.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:58 smithi083 ceph-mon[24810]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-07T14:34:58.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:58 smithi083 ceph-mon[24810]: from='osd.3 [v2:172.21.15.83:6826/3079630945,v1:172.21.15.83:6827/3079630945]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2024-04-07T14:34:58.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:58 smithi083 ceph-mon[24810]: osdmap e25: 4 total, 3 up, 4 in 2024-04-07T14:34:58.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:58 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-07T14:34:58.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:58 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-07T14:34:58.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:58 smithi083 ceph-mon[29774]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-07T14:34:58.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:58 smithi083 ceph-mon[29774]: from='osd.3 [v2:172.21.15.83:6826/3079630945,v1:172.21.15.83:6827/3079630945]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2024-04-07T14:34:58.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:58 smithi083 ceph-mon[29774]: osdmap e25: 4 total, 3 up, 4 in 2024-04-07T14:34:58.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:58 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-07T14:34:58.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:58 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-07T14:34:58.444 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:58 smithi189 ceph-mon[25797]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-07T14:34:58.444 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:58 smithi189 ceph-mon[25797]: from='osd.3 [v2:172.21.15.83:6826/3079630945,v1:172.21.15.83:6827/3079630945]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi083", "root=default"]}]': finished 2024-04-07T14:34:58.444 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:58 smithi189 ceph-mon[25797]: osdmap e25: 4 total, 3 up, 4 in 2024-04-07T14:34:58.444 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:58 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-07T14:34:58.444 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:58 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-07T14:34:58.599 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph orch daemon add osd smithi189:vg_nvme/lv_4 2024-04-07T14:34:59.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:59 smithi083 ceph-mon[24810]: purged_snaps scrub starts 2024-04-07T14:34:59.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:59 smithi083 ceph-mon[24810]: purged_snaps scrub ok 2024-04-07T14:34:59.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:59 smithi083 ceph-mon[24810]: osd.3 [v2:172.21.15.83:6826/3079630945,v1:172.21.15.83:6827/3079630945] boot 2024-04-07T14:34:59.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:59 smithi083 ceph-mon[24810]: osdmap e26: 4 total, 4 up, 4 in 2024-04-07T14:34:59.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:59 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-07T14:34:59.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:34:59 smithi083 ceph-mon[24810]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-07T14:34:59.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:59 smithi083 ceph-mon[29774]: purged_snaps scrub starts 2024-04-07T14:34:59.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:59 smithi083 ceph-mon[29774]: purged_snaps scrub ok 2024-04-07T14:34:59.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:59 smithi083 ceph-mon[29774]: osd.3 [v2:172.21.15.83:6826/3079630945,v1:172.21.15.83:6827/3079630945] boot 2024-04-07T14:34:59.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:59 smithi083 ceph-mon[29774]: osdmap e26: 4 total, 4 up, 4 in 2024-04-07T14:34:59.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:59 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-07T14:34:59.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:34:59 smithi083 ceph-mon[29774]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-07T14:34:59.703 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:59 smithi189 ceph-mon[25797]: purged_snaps scrub starts 2024-04-07T14:34:59.704 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:59 smithi189 ceph-mon[25797]: purged_snaps scrub ok 2024-04-07T14:34:59.704 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:59 smithi189 ceph-mon[25797]: osd.3 [v2:172.21.15.83:6826/3079630945,v1:172.21.15.83:6827/3079630945] boot 2024-04-07T14:34:59.704 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:59 smithi189 ceph-mon[25797]: osdmap e26: 4 total, 4 up, 4 in 2024-04-07T14:34:59.704 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:59 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-07T14:34:59.704 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:34:59 smithi189 ceph-mon[25797]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-07T14:35:00.176 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:34:59 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:34:59] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:35:00.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:00 smithi189 ceph-mon[25797]: osdmap e27: 4 total, 4 up, 4 in 2024-04-07T14:35:00.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:00 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:00.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:00 smithi189 ceph-mon[25797]: Detected new or changed devices on smithi083 2024-04-07T14:35:00.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:00 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:00.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:00 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-07T14:35:00.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:00 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-07T14:35:00.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:00 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:00.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:00 smithi083 ceph-mon[29774]: osdmap e27: 4 total, 4 up, 4 in 2024-04-07T14:35:00.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:00 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:00.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:00 smithi083 ceph-mon[29774]: Detected new or changed devices on smithi083 2024-04-07T14:35:00.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:00 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:00.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:00 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-07T14:35:00.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:00 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-07T14:35:00.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:00 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:00.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:00 smithi083 ceph-mon[24810]: osdmap e27: 4 total, 4 up, 4 in 2024-04-07T14:35:00.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:00 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:00.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:00 smithi083 ceph-mon[24810]: Detected new or changed devices on smithi083 2024-04-07T14:35:00.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:00 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:00.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:00 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-07T14:35:00.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:00 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-07T14:35:00.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:00 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:01.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:01 smithi189 ceph-mon[25797]: from='client.24197 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:35:01.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:01 smithi189 ceph-mon[25797]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-07T14:35:01.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:01 smithi083 ceph-mon[24810]: from='client.24197 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:35:01.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:01 smithi083 ceph-mon[24810]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-07T14:35:01.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:01 smithi083 ceph-mon[29774]: from='client.24197 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:35:01.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:01 smithi083 ceph-mon[29774]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-07T14:35:02.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:02 smithi083 ceph-mon[29774]: from='client.? 172.21.15.189:0/1953278638' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6a8687be-8c65-40a3-89c4-d463ed60d920"}]: dispatch 2024-04-07T14:35:02.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:02 smithi083 ceph-mon[29774]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6a8687be-8c65-40a3-89c4-d463ed60d920"}]: dispatch 2024-04-07T14:35:02.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:02 smithi083 ceph-mon[29774]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6a8687be-8c65-40a3-89c4-d463ed60d920"}]': finished 2024-04-07T14:35:02.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:02 smithi083 ceph-mon[29774]: osdmap e28: 5 total, 4 up, 5 in 2024-04-07T14:35:02.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:02 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-07T14:35:02.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:02 smithi083 ceph-mon[24810]: from='client.? 172.21.15.189:0/1953278638' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6a8687be-8c65-40a3-89c4-d463ed60d920"}]: dispatch 2024-04-07T14:35:02.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:02 smithi083 ceph-mon[24810]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6a8687be-8c65-40a3-89c4-d463ed60d920"}]: dispatch 2024-04-07T14:35:02.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:02 smithi083 ceph-mon[24810]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6a8687be-8c65-40a3-89c4-d463ed60d920"}]': finished 2024-04-07T14:35:02.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:02 smithi083 ceph-mon[24810]: osdmap e28: 5 total, 4 up, 5 in 2024-04-07T14:35:02.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:02 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-07T14:35:02.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:02 smithi189 ceph-mon[25797]: from='client.? 172.21.15.189:0/1953278638' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6a8687be-8c65-40a3-89c4-d463ed60d920"}]: dispatch 2024-04-07T14:35:02.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:02 smithi189 ceph-mon[25797]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6a8687be-8c65-40a3-89c4-d463ed60d920"}]: dispatch 2024-04-07T14:35:02.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:02 smithi189 ceph-mon[25797]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6a8687be-8c65-40a3-89c4-d463ed60d920"}]': finished 2024-04-07T14:35:02.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:02 smithi189 ceph-mon[25797]: osdmap e28: 5 total, 4 up, 5 in 2024-04-07T14:35:02.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:02 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-07T14:35:03.604 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:35:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:35:03] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:35:03.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:03 smithi083 ceph-mon[29774]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-07T14:35:03.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:03 smithi083 ceph-mon[29774]: from='client.? 172.21.15.189:0/1660738953' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-07T14:35:03.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:03 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:03.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:03 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:03.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:03 smithi083 ceph-mon[24810]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-07T14:35:03.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:03 smithi083 ceph-mon[24810]: from='client.? 172.21.15.189:0/1660738953' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-07T14:35:03.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:03 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:03.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:03 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:03.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:03 smithi189 ceph-mon[25797]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-07T14:35:03.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:03 smithi189 ceph-mon[25797]: from='client.? 172.21.15.189:0/1660738953' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-07T14:35:03.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:03 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:03.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:03 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:06.285 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:06 smithi189 ceph-mon[25797]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-07T14:35:06.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:06 smithi083 ceph-mon[24810]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-07T14:35:06.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:06 smithi083 ceph-mon[29774]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-07T14:35:08.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:08 smithi189 ceph-mon[25797]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-07T14:35:08.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:08 smithi083 ceph-mon[29774]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-07T14:35:08.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:08 smithi083 ceph-mon[24810]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-07T14:35:10.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:10 smithi083 ceph-mon[24810]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-07T14:35:10.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:10 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-07T14:35:10.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:10 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:10.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:10 smithi083 ceph-mon[29774]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-07T14:35:10.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:10 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-07T14:35:10.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:10 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:10.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:10 smithi189 ceph-mon[25797]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-07T14:35:10.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:10 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-07T14:35:10.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:10 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:11.313 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:11 smithi189 ceph-mon[25797]: Deploying daemon osd.4 on smithi189 2024-04-07T14:35:11.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:11 smithi083 ceph-mon[24810]: Deploying daemon osd.4 on smithi189 2024-04-07T14:35:11.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:11 smithi083 ceph-mon[29774]: Deploying daemon osd.4 on smithi189 2024-04-07T14:35:12.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:12 smithi189 ceph-mon[25797]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-07T14:35:12.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:12 smithi083 ceph-mon[24810]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-07T14:35:12.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:12 smithi083 ceph-mon[29774]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-07T14:35:14.347 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:14 smithi189 ceph-mon[25797]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-07T14:35:14.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:14 smithi083 ceph-mon[24810]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-07T14:35:14.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:14 smithi083 ceph-mon[29774]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-07T14:35:15.176 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:35:14 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:35:14] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:35:15.563 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:15 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:15.563 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:15 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:35:15.563 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:15 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:15.563 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:15 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:35:15.563 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:15 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:15.564 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:15 smithi083 ceph-mon[24810]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-07T14:35:15.564 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:15 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:15.564 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:15 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:15.564 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:15 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:15.564 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:15 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:35:15.564 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:15 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:15.564 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:15 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:35:15.564 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:15 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:15.564 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:15 smithi083 ceph-mon[29774]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-07T14:35:15.564 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:15 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:15.564 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:15 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:15.704 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:15 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:15.704 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:15 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:35:15.704 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:15 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:15.704 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:15 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:35:15.704 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:15 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:15.704 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:15 smithi189 ceph-mon[25797]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-07T14:35:15.705 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:15 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:15.705 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:15 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:16.891 INFO:teuthology.orchestra.run.smithi189.stdout:Created osd(s) 4 on host 'smithi189' 2024-04-07T14:35:17.522 DEBUG:teuthology.orchestra.run.smithi189:osd.4> sudo journalctl -f -n 0 -u ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@osd.4.service 2024-04-07T14:35:17.525 INFO:tasks.cephadm:Deploying osd.5 on smithi189 with /dev/vg_nvme/lv_3... 2024-04-07T14:35:17.525 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-07T14:35:18.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:17 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:18.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:17 smithi083 ceph-mon[24810]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-07T14:35:18.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:17 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:35:18.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:17 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:18.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:17 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:35:18.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:17 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:35:18.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:17 smithi083 ceph-mon[24810]: Adjusting osd_memory_target on smithi189 to 16064M 2024-04-07T14:35:18.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:17 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:18.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:17 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:18.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:17 smithi083 ceph-mon[24810]: from='osd.4 [v2:172.21.15.189:6800/3355623828,v1:172.21.15.189:6801/3355623828]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-07T14:35:18.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:17 smithi083 ceph-mon[24810]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-07T14:35:18.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:17 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:18.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:17 smithi083 ceph-mon[29774]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-07T14:35:18.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:17 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:35:18.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:17 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:18.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:17 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:35:18.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:17 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:35:18.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:17 smithi083 ceph-mon[29774]: Adjusting osd_memory_target on smithi189 to 16064M 2024-04-07T14:35:18.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:17 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:18.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:17 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:18.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:17 smithi083 ceph-mon[29774]: from='osd.4 [v2:172.21.15.189:6800/3355623828,v1:172.21.15.189:6801/3355623828]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-07T14:35:18.178 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:17 smithi083 ceph-mon[29774]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-07T14:35:18.201 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:17 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:18.201 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:17 smithi189 ceph-mon[25797]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-07T14:35:18.201 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:17 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:35:18.201 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:17 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:18.201 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:17 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:35:18.201 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:17 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:35:18.201 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:17 smithi189 ceph-mon[25797]: Adjusting osd_memory_target on smithi189 to 16064M 2024-04-07T14:35:18.201 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:17 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:18.201 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:17 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:18.201 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:17 smithi189 ceph-mon[25797]: from='osd.4 [v2:172.21.15.189:6800/3355623828,v1:172.21.15.189:6801/3355623828]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-07T14:35:18.201 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:17 smithi189 ceph-mon[25797]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-07T14:35:19.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:18 smithi189 ceph-mon[25797]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-07T14:35:19.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:18 smithi189 ceph-mon[25797]: osdmap e29: 5 total, 4 up, 5 in 2024-04-07T14:35:19.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:18 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-07T14:35:19.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:18 smithi189 ceph-mon[25797]: from='osd.4 [v2:172.21.15.189:6800/3355623828,v1:172.21.15.189:6801/3355623828]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-04-07T14:35:19.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:18 smithi189 ceph-mon[25797]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-04-07T14:35:19.209 INFO:journalctl@ceph.osd.4.smithi189.stdout:Apr 07 14:35:18 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-4[30531]: 2024-04-07T14:35:18.953+0000 7f95380fc700 -1 osd.4 0 waiting for initial osdmap 2024-04-07T14:35:19.209 INFO:journalctl@ceph.osd.4.smithi189.stdout:Apr 07 14:35:18 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-4[30531]: 2024-04-07T14:35:18.957+0000 7f9531290700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-07T14:35:19.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:18 smithi083 ceph-mon[29774]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-07T14:35:19.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:18 smithi083 ceph-mon[29774]: osdmap e29: 5 total, 4 up, 5 in 2024-04-07T14:35:19.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:18 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-07T14:35:19.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:18 smithi083 ceph-mon[29774]: from='osd.4 [v2:172.21.15.189:6800/3355623828,v1:172.21.15.189:6801/3355623828]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-04-07T14:35:19.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:18 smithi083 ceph-mon[29774]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-04-07T14:35:19.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:18 smithi083 ceph-mon[24810]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-07T14:35:19.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:18 smithi083 ceph-mon[24810]: osdmap e29: 5 total, 4 up, 5 in 2024-04-07T14:35:19.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:18 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-07T14:35:19.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:18 smithi083 ceph-mon[24810]: from='osd.4 [v2:172.21.15.189:6800/3355623828,v1:172.21.15.189:6801/3355623828]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-04-07T14:35:19.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:18 smithi083 ceph-mon[24810]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-04-07T14:35:19.535 INFO:teuthology.orchestra.run.smithi189.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-07T14:35:19.536 INFO:teuthology.orchestra.run.smithi189.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-07T14:35:19.536 INFO:teuthology.orchestra.run.smithi189.stderr: stderr: 10+0 records in 2024-04-07T14:35:19.536 INFO:teuthology.orchestra.run.smithi189.stderr:10+0 records out 2024-04-07T14:35:19.536 INFO:teuthology.orchestra.run.smithi189.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0125431 s, 836 MB/s 2024-04-07T14:35:19.536 INFO:teuthology.orchestra.run.smithi189.stderr:--> Zapping successful for: 2024-04-07T14:35:20.125 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph orch daemon add osd smithi189:vg_nvme/lv_3 2024-04-07T14:35:20.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:19 smithi189 ceph-mon[25797]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-07T14:35:20.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:19 smithi189 ceph-mon[25797]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-04-07T14:35:20.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:19 smithi189 ceph-mon[25797]: osdmap e30: 5 total, 4 up, 5 in 2024-04-07T14:35:20.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:19 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-07T14:35:20.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:19 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-07T14:35:20.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:19 smithi083 ceph-mon[24810]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-07T14:35:20.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:19 smithi083 ceph-mon[24810]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-04-07T14:35:20.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:19 smithi083 ceph-mon[24810]: osdmap e30: 5 total, 4 up, 5 in 2024-04-07T14:35:20.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:19 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-07T14:35:20.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:19 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-07T14:35:20.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:19 smithi083 ceph-mon[29774]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-07T14:35:20.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:19 smithi083 ceph-mon[29774]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-04-07T14:35:20.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:19 smithi083 ceph-mon[29774]: osdmap e30: 5 total, 4 up, 5 in 2024-04-07T14:35:20.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:19 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-07T14:35:20.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:19 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-07T14:35:21.116 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:20 smithi189 ceph-mon[25797]: purged_snaps scrub starts 2024-04-07T14:35:21.116 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:20 smithi189 ceph-mon[25797]: purged_snaps scrub ok 2024-04-07T14:35:21.116 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:20 smithi189 ceph-mon[25797]: osd.4 [v2:172.21.15.189:6800/3355623828,v1:172.21.15.189:6801/3355623828] boot 2024-04-07T14:35:21.116 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:20 smithi189 ceph-mon[25797]: osdmap e31: 5 total, 5 up, 5 in 2024-04-07T14:35:21.117 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:20 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-07T14:35:21.117 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:20 smithi189 ceph-mon[25797]: osdmap e32: 5 total, 5 up, 5 in 2024-04-07T14:35:21.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:20 smithi083 ceph-mon[24810]: purged_snaps scrub starts 2024-04-07T14:35:21.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:20 smithi083 ceph-mon[24810]: purged_snaps scrub ok 2024-04-07T14:35:21.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:20 smithi083 ceph-mon[24810]: osd.4 [v2:172.21.15.189:6800/3355623828,v1:172.21.15.189:6801/3355623828] boot 2024-04-07T14:35:21.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:20 smithi083 ceph-mon[24810]: osdmap e31: 5 total, 5 up, 5 in 2024-04-07T14:35:21.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:20 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-07T14:35:21.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:20 smithi083 ceph-mon[24810]: osdmap e32: 5 total, 5 up, 5 in 2024-04-07T14:35:21.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:20 smithi083 ceph-mon[29774]: purged_snaps scrub starts 2024-04-07T14:35:21.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:20 smithi083 ceph-mon[29774]: purged_snaps scrub ok 2024-04-07T14:35:21.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:20 smithi083 ceph-mon[29774]: osd.4 [v2:172.21.15.189:6800/3355623828,v1:172.21.15.189:6801/3355623828] boot 2024-04-07T14:35:21.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:20 smithi083 ceph-mon[29774]: osdmap e31: 5 total, 5 up, 5 in 2024-04-07T14:35:21.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:20 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-07T14:35:21.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:20 smithi083 ceph-mon[29774]: osdmap e32: 5 total, 5 up, 5 in 2024-04-07T14:35:22.270 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:22 smithi189 ceph-mon[25797]: pgmap v113: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-07T14:35:22.270 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:22 smithi189 ceph-mon[25797]: osdmap e33: 5 total, 5 up, 5 in 2024-04-07T14:35:22.270 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:22 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-07T14:35:22.270 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:22 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-07T14:35:22.271 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:22 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:22.271 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:22 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:35:22.271 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:22 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:35:22.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:22 smithi083 ceph-mon[24810]: pgmap v113: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-07T14:35:22.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:22 smithi083 ceph-mon[24810]: osdmap e33: 5 total, 5 up, 5 in 2024-04-07T14:35:22.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:22 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-07T14:35:22.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:22 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-07T14:35:22.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:22 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:22.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:22 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:35:22.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:22 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:35:22.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:22 smithi083 ceph-mon[29774]: pgmap v113: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-07T14:35:22.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:22 smithi083 ceph-mon[29774]: osdmap e33: 5 total, 5 up, 5 in 2024-04-07T14:35:22.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:22 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-07T14:35:22.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:22 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-07T14:35:22.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:22 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:22.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:22 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:35:22.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:22 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:35:23.316 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:23 smithi189 ceph-mon[25797]: from='client.24224 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:35:23.316 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:23 smithi189 ceph-mon[25797]: osdmap e34: 5 total, 5 up, 5 in 2024-04-07T14:35:23.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:23 smithi083 ceph-mon[24810]: from='client.24224 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:35:23.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:23 smithi083 ceph-mon[24810]: osdmap e34: 5 total, 5 up, 5 in 2024-04-07T14:35:23.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:23 smithi083 ceph-mon[29774]: from='client.24224 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:35:23.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:23 smithi083 ceph-mon[29774]: osdmap e34: 5 total, 5 up, 5 in 2024-04-07T14:35:23.676 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:35:23 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:35:23] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:35:24.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:24 smithi189 ceph-mon[25797]: pgmap v116: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-07T14:35:24.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:24 smithi189 ceph-mon[25797]: osdmap e35: 5 total, 5 up, 5 in 2024-04-07T14:35:24.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:24 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:24.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:24 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:24.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:24 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:35:24.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:24 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:24.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:24 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:35:24.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:24 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:24.653 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:24 smithi083 ceph-mon[29774]: pgmap v116: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-07T14:35:24.653 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:24 smithi083 ceph-mon[29774]: osdmap e35: 5 total, 5 up, 5 in 2024-04-07T14:35:24.653 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:24 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:24.653 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:24 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:24.653 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:24 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:35:24.653 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:24 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:24.653 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:24 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:35:24.653 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:24 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:24.653 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:24 smithi083 ceph-mon[24810]: pgmap v116: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-07T14:35:24.653 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:24 smithi083 ceph-mon[24810]: osdmap e35: 5 total, 5 up, 5 in 2024-04-07T14:35:24.653 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:24 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:24.654 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:24 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:24.654 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:24 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:35:24.654 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:24 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:24.654 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:24 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:35:24.654 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:24 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:24.926 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:35:24 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:35:24] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:35:25.530 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:25 smithi189 ceph-mon[25797]: Detected new or changed devices on smithi189 2024-04-07T14:35:25.530 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:25 smithi189 ceph-mon[25797]: pgmap v118: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-07T14:35:25.530 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:25 smithi189 ceph-mon[25797]: from='client.? 172.21.15.189:0/2402383909' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7115ab07-1339-4eac-b403-bf6b40361af9"}]: dispatch 2024-04-07T14:35:25.530 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:25 smithi189 ceph-mon[25797]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7115ab07-1339-4eac-b403-bf6b40361af9"}]: dispatch 2024-04-07T14:35:25.530 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:25 smithi189 ceph-mon[25797]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7115ab07-1339-4eac-b403-bf6b40361af9"}]': finished 2024-04-07T14:35:25.530 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:25 smithi189 ceph-mon[25797]: osdmap e36: 6 total, 5 up, 6 in 2024-04-07T14:35:25.530 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:25 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-07T14:35:25.530 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:25 smithi189 ceph-mon[25797]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-04-07T14:35:25.530 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:25 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:25.530 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:25 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:25.530 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:25 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:35:25.531 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:25 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:25.531 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:25 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:35:25.531 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:25 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:25.531 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:25 smithi189 ceph-mon[25797]: from='client.? 172.21.15.189:0/261442292' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-07T14:35:25.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:25 smithi083 ceph-mon[24810]: Detected new or changed devices on smithi189 2024-04-07T14:35:25.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:25 smithi083 ceph-mon[24810]: pgmap v118: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-07T14:35:25.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:25 smithi083 ceph-mon[24810]: from='client.? 172.21.15.189:0/2402383909' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7115ab07-1339-4eac-b403-bf6b40361af9"}]: dispatch 2024-04-07T14:35:25.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:25 smithi083 ceph-mon[24810]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7115ab07-1339-4eac-b403-bf6b40361af9"}]: dispatch 2024-04-07T14:35:25.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:25 smithi083 ceph-mon[24810]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7115ab07-1339-4eac-b403-bf6b40361af9"}]': finished 2024-04-07T14:35:25.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:25 smithi083 ceph-mon[24810]: osdmap e36: 6 total, 5 up, 6 in 2024-04-07T14:35:25.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:25 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-07T14:35:25.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:25 smithi083 ceph-mon[24810]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-04-07T14:35:25.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:25 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:25.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:25 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:25.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:25 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:35:25.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:25 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:25.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:25 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:35:25.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:25 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:25.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:25 smithi083 ceph-mon[24810]: from='client.? 172.21.15.189:0/261442292' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-07T14:35:25.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:25 smithi083 ceph-mon[29774]: Detected new or changed devices on smithi189 2024-04-07T14:35:25.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:25 smithi083 ceph-mon[29774]: pgmap v118: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-07T14:35:25.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:25 smithi083 ceph-mon[29774]: from='client.? 172.21.15.189:0/2402383909' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7115ab07-1339-4eac-b403-bf6b40361af9"}]: dispatch 2024-04-07T14:35:25.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:25 smithi083 ceph-mon[29774]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7115ab07-1339-4eac-b403-bf6b40361af9"}]: dispatch 2024-04-07T14:35:25.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:25 smithi083 ceph-mon[29774]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7115ab07-1339-4eac-b403-bf6b40361af9"}]': finished 2024-04-07T14:35:25.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:25 smithi083 ceph-mon[29774]: osdmap e36: 6 total, 5 up, 6 in 2024-04-07T14:35:25.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:25 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-07T14:35:25.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:25 smithi083 ceph-mon[29774]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-04-07T14:35:25.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:25 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:25.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:25 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:25.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:25 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:35:25.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:25 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:25.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:25 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:35:25.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:25 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:25.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:25 smithi083 ceph-mon[29774]: from='client.? 172.21.15.189:0/261442292' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-07T14:35:26.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:26 smithi083 ceph-mon[24810]: pgmap v120: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-07T14:35:26.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:26 smithi083 ceph-mon[24810]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-04-07T14:35:26.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:26 smithi083 ceph-mon[24810]: Cluster is now healthy 2024-04-07T14:35:26.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:26 smithi083 ceph-mon[29774]: pgmap v120: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-07T14:35:26.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:26 smithi083 ceph-mon[29774]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-04-07T14:35:26.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:26 smithi083 ceph-mon[29774]: Cluster is now healthy 2024-04-07T14:35:26.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:26 smithi189 ceph-mon[25797]: pgmap v120: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-07T14:35:26.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:26 smithi189 ceph-mon[25797]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-04-07T14:35:26.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:26 smithi189 ceph-mon[25797]: Cluster is now healthy 2024-04-07T14:35:28.399 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:28 smithi189 ceph-mon[25797]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 117 KiB/s, 0 objects/s recovering 2024-04-07T14:35:28.564 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:28 smithi083 ceph-mon[29774]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 117 KiB/s, 0 objects/s recovering 2024-04-07T14:35:28.565 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:28 smithi083 ceph-mon[24810]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 117 KiB/s, 0 objects/s recovering 2024-04-07T14:35:30.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:30 smithi189 ceph-mon[25797]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 90 KiB/s, 0 objects/s recovering 2024-04-07T14:35:30.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:30 smithi083 ceph-mon[24810]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 90 KiB/s, 0 objects/s recovering 2024-04-07T14:35:30.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:30 smithi083 ceph-mon[29774]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 90 KiB/s, 0 objects/s recovering 2024-04-07T14:35:31.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:31 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-07T14:35:31.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:31 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:31.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:31 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-07T14:35:31.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:31 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:31.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:31 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-07T14:35:31.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:31 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:32.544 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:32 smithi189 ceph-mon[25797]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-04-07T14:35:32.544 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:32 smithi189 ceph-mon[25797]: Deploying daemon osd.5 on smithi189 2024-04-07T14:35:32.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:32 smithi083 ceph-mon[24810]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-04-07T14:35:32.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:32 smithi083 ceph-mon[24810]: Deploying daemon osd.5 on smithi189 2024-04-07T14:35:32.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:32 smithi083 ceph-mon[29774]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-04-07T14:35:32.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:32 smithi083 ceph-mon[29774]: Deploying daemon osd.5 on smithi189 2024-04-07T14:35:34.361 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:34 smithi083 ceph-mon[24810]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 55 KiB/s, 0 objects/s recovering 2024-04-07T14:35:34.361 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:34 smithi083 ceph-mon[29774]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 55 KiB/s, 0 objects/s recovering 2024-04-07T14:35:34.408 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:34 smithi189 ceph-mon[25797]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 55 KiB/s, 0 objects/s recovering 2024-04-07T14:35:35.395 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:35:35 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:35:35] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:35:36.136 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:36 smithi189 ceph-mon[25797]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 50 KiB/s, 0 objects/s recovering 2024-04-07T14:35:36.136 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:36 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:36.136 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:36 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:36.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:36 smithi083 ceph-mon[29774]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 50 KiB/s, 0 objects/s recovering 2024-04-07T14:35:36.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:36 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:36.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:36 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:36.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:36 smithi083 ceph-mon[24810]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 50 KiB/s, 0 objects/s recovering 2024-04-07T14:35:36.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:36 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:36.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:36 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:37.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:37 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:37.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:37 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:35:37.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:37 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:37.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:37 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:35:37.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:37 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:37.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:37 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:37.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:37 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:35:37.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:37 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:37.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:37 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:35:37.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:37 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:37.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:37 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:37.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:37 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:35:37.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:37 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:37.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:37 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:35:37.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:37 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:38.417 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:38 smithi189 ceph-mon[25797]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-04-07T14:35:38.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:38 smithi083 ceph-mon[24810]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-04-07T14:35:38.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:38 smithi083 ceph-mon[29774]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-04-07T14:35:38.706 INFO:teuthology.orchestra.run.smithi189.stdout:Created osd(s) 5 on host 'smithi189' 2024-04-07T14:35:39.385 DEBUG:teuthology.orchestra.run.smithi189:osd.5> sudo journalctl -f -n 0 -u ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@osd.5.service 2024-04-07T14:35:39.387 INFO:tasks.cephadm:Deploying osd.6 on smithi189 with /dev/vg_nvme/lv_2... 2024-04-07T14:35:39.388 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-07T14:35:39.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:39 smithi189 ceph-mon[25797]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-07T14:35:39.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:39 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:39.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:39 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:35:39.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:39 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:39.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:39 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:35:39.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:39 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:35:39.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:39 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:35:39.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:39 smithi189 ceph-mon[25797]: Adjusting osd_memory_target on smithi189 to 8032M 2024-04-07T14:35:39.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:39 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:39.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:39 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:39.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:39 smithi189 ceph-mon[25797]: from='osd.5 [v2:172.21.15.189:6808/3207731440,v1:172.21.15.189:6809/3207731440]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-07T14:35:39.960 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:39 smithi189 ceph-mon[25797]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-07T14:35:40.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:39 smithi083 ceph-mon[24810]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-07T14:35:40.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:39 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:40.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:39 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:35:40.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:39 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:40.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:39 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:35:40.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:39 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:35:40.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:39 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:35:40.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:39 smithi083 ceph-mon[24810]: Adjusting osd_memory_target on smithi189 to 8032M 2024-04-07T14:35:40.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:39 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:40.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:39 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:40.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:39 smithi083 ceph-mon[24810]: from='osd.5 [v2:172.21.15.189:6808/3207731440,v1:172.21.15.189:6809/3207731440]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-07T14:35:40.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:39 smithi083 ceph-mon[24810]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-07T14:35:40.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:39 smithi083 ceph-mon[29774]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-07T14:35:40.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:39 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:40.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:39 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:35:40.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:39 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:40.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:39 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:35:40.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:39 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:35:40.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:39 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:35:40.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:39 smithi083 ceph-mon[29774]: Adjusting osd_memory_target on smithi189 to 8032M 2024-04-07T14:35:40.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:39 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:40.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:39 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:40.178 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:39 smithi083 ceph-mon[29774]: from='osd.5 [v2:172.21.15.189:6808/3207731440,v1:172.21.15.189:6809/3207731440]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-07T14:35:40.178 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:39 smithi083 ceph-mon[29774]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-07T14:35:41.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:41 smithi189 ceph-mon[25797]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-07T14:35:41.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:41 smithi189 ceph-mon[25797]: osdmap e37: 6 total, 5 up, 6 in 2024-04-07T14:35:41.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:41 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-07T14:35:41.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:41 smithi189 ceph-mon[25797]: from='osd.5 [v2:172.21.15.189:6808/3207731440,v1:172.21.15.189:6809/3207731440]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-04-07T14:35:41.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:41 smithi189 ceph-mon[25797]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-04-07T14:35:41.209 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:35:41 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5[33437]: 2024-04-07T14:35:41.148+0000 7fd19845d700 -1 osd.5 0 waiting for initial osdmap 2024-04-07T14:35:41.209 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:35:41 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5[33437]: 2024-04-07T14:35:41.155+0000 7fd192df4700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-07T14:35:41.393 INFO:teuthology.orchestra.run.smithi189.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-07T14:35:41.393 INFO:teuthology.orchestra.run.smithi189.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-07T14:35:41.393 INFO:teuthology.orchestra.run.smithi189.stderr: stderr: 10+0 records in 2024-04-07T14:35:41.393 INFO:teuthology.orchestra.run.smithi189.stderr:10+0 records out 2024-04-07T14:35:41.393 INFO:teuthology.orchestra.run.smithi189.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.012379 s, 847 MB/s 2024-04-07T14:35:41.393 INFO:teuthology.orchestra.run.smithi189.stderr:--> Zapping successful for: 2024-04-07T14:35:41.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:41 smithi083 ceph-mon[24810]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-07T14:35:41.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:41 smithi083 ceph-mon[24810]: osdmap e37: 6 total, 5 up, 6 in 2024-04-07T14:35:41.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:41 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-07T14:35:41.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:41 smithi083 ceph-mon[24810]: from='osd.5 [v2:172.21.15.189:6808/3207731440,v1:172.21.15.189:6809/3207731440]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-04-07T14:35:41.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:41 smithi083 ceph-mon[24810]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-04-07T14:35:41.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:41 smithi083 ceph-mon[29774]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-07T14:35:41.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:41 smithi083 ceph-mon[29774]: osdmap e37: 6 total, 5 up, 6 in 2024-04-07T14:35:41.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:41 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-07T14:35:41.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:41 smithi083 ceph-mon[29774]: from='osd.5 [v2:172.21.15.189:6808/3207731440,v1:172.21.15.189:6809/3207731440]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-04-07T14:35:41.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:41 smithi083 ceph-mon[29774]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-04-07T14:35:42.043 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph orch daemon add osd smithi189:vg_nvme/lv_2 2024-04-07T14:35:42.334 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:42 smithi189 ceph-mon[25797]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-07T14:35:42.335 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:42 smithi189 ceph-mon[25797]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-04-07T14:35:42.335 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:42 smithi189 ceph-mon[25797]: osdmap e38: 6 total, 5 up, 6 in 2024-04-07T14:35:42.335 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:42 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-07T14:35:42.335 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:42 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-07T14:35:42.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:42 smithi083 ceph-mon[24810]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-07T14:35:42.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:42 smithi083 ceph-mon[24810]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-04-07T14:35:42.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:42 smithi083 ceph-mon[24810]: osdmap e38: 6 total, 5 up, 6 in 2024-04-07T14:35:42.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:42 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-07T14:35:42.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:42 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-07T14:35:42.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:42 smithi083 ceph-mon[29774]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-07T14:35:42.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:42 smithi083 ceph-mon[29774]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-04-07T14:35:42.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:42 smithi083 ceph-mon[29774]: osdmap e38: 6 total, 5 up, 6 in 2024-04-07T14:35:42.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:42 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-07T14:35:42.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:42 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-07T14:35:43.403 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:43 smithi189 ceph-mon[25797]: purged_snaps scrub starts 2024-04-07T14:35:43.403 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:43 smithi189 ceph-mon[25797]: purged_snaps scrub ok 2024-04-07T14:35:43.403 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:43 smithi189 ceph-mon[25797]: osd.5 [v2:172.21.15.189:6808/3207731440,v1:172.21.15.189:6809/3207731440] boot 2024-04-07T14:35:43.403 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:43 smithi189 ceph-mon[25797]: osdmap e39: 6 total, 6 up, 6 in 2024-04-07T14:35:43.403 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:43 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-07T14:35:43.403 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:43 smithi189 ceph-mon[25797]: osdmap e40: 6 total, 6 up, 6 in 2024-04-07T14:35:43.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:43 smithi083 ceph-mon[24810]: purged_snaps scrub starts 2024-04-07T14:35:43.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:43 smithi083 ceph-mon[24810]: purged_snaps scrub ok 2024-04-07T14:35:43.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:43 smithi083 ceph-mon[24810]: osd.5 [v2:172.21.15.189:6808/3207731440,v1:172.21.15.189:6809/3207731440] boot 2024-04-07T14:35:43.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:43 smithi083 ceph-mon[24810]: osdmap e39: 6 total, 6 up, 6 in 2024-04-07T14:35:43.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:43 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-07T14:35:43.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:43 smithi083 ceph-mon[24810]: osdmap e40: 6 total, 6 up, 6 in 2024-04-07T14:35:43.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:43 smithi083 ceph-mon[29774]: purged_snaps scrub starts 2024-04-07T14:35:43.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:43 smithi083 ceph-mon[29774]: purged_snaps scrub ok 2024-04-07T14:35:43.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:43 smithi083 ceph-mon[29774]: osd.5 [v2:172.21.15.189:6808/3207731440,v1:172.21.15.189:6809/3207731440] boot 2024-04-07T14:35:43.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:43 smithi083 ceph-mon[29774]: osdmap e39: 6 total, 6 up, 6 in 2024-04-07T14:35:43.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:43 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-07T14:35:43.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:43 smithi083 ceph-mon[29774]: osdmap e40: 6 total, 6 up, 6 in 2024-04-07T14:35:44.419 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:44 smithi189 ceph-mon[25797]: pgmap v132: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-07T14:35:44.419 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:44 smithi189 ceph-mon[25797]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-07T14:35:44.419 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:44 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-07T14:35:44.419 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:44 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-07T14:35:44.419 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:44 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:44.420 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:44 smithi189 ceph-mon[25797]: osdmap e41: 6 total, 6 up, 6 in 2024-04-07T14:35:44.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:44 smithi083 ceph-mon[24810]: pgmap v132: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-07T14:35:44.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:44 smithi083 ceph-mon[24810]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-07T14:35:44.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:44 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-07T14:35:44.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:44 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-07T14:35:44.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:44 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:44.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:44 smithi083 ceph-mon[24810]: osdmap e41: 6 total, 6 up, 6 in 2024-04-07T14:35:44.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:44 smithi083 ceph-mon[29774]: pgmap v132: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-07T14:35:44.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:44 smithi083 ceph-mon[29774]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-07T14:35:44.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:44 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-07T14:35:44.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:44 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-07T14:35:44.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:44 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:44.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:44 smithi083 ceph-mon[29774]: osdmap e41: 6 total, 6 up, 6 in 2024-04-07T14:35:45.040 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:35:44 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:35:44] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:35:45.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:45 smithi083 ceph-mon[29774]: from='client.24251 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:35:45.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:45 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:45.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:45 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:45.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:45 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:35:45.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:45 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:45.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:45 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:35:45.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:45 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:45.428 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:45 smithi083 ceph-mon[24810]: from='client.24251 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:35:45.428 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:45 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:45.428 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:45 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:45.428 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:45 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:35:45.429 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:45 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:45.429 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:45 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:35:45.429 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:45 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:45.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:45 smithi189 ceph-mon[25797]: from='client.24251 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:35:45.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:45 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:45.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:45 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:45.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:45 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:35:45.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:45 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:45.460 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:45 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:35:45.460 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:45 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:45.926 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:35:45 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:35:45] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:35:46.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:46 smithi083 ceph-mon[29774]: pgmap v135: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-07T14:35:46.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:46 smithi083 ceph-mon[29774]: Detected new or changed devices on smithi189 2024-04-07T14:35:46.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:46 smithi083 ceph-mon[29774]: pgmap v136: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-07T14:35:46.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:46 smithi083 ceph-mon[29774]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-04-07T14:35:46.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:46 smithi083 ceph-mon[29774]: osdmap e42: 6 total, 6 up, 6 in 2024-04-07T14:35:46.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:46 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:46.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:46 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:46.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:46 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:35:46.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:46 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:46.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:46 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:35:46.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:46 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:46.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:46 smithi083 ceph-mon[24810]: pgmap v135: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-07T14:35:46.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:46 smithi083 ceph-mon[24810]: Detected new or changed devices on smithi189 2024-04-07T14:35:46.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:46 smithi083 ceph-mon[24810]: pgmap v136: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-07T14:35:46.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:46 smithi083 ceph-mon[24810]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-04-07T14:35:46.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:46 smithi083 ceph-mon[24810]: osdmap e42: 6 total, 6 up, 6 in 2024-04-07T14:35:46.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:46 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:46.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:46 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:46.428 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:46 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:35:46.428 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:46 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:46.428 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:46 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:35:46.428 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:46 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:46.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:46 smithi189 ceph-mon[25797]: pgmap v135: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-07T14:35:46.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:46 smithi189 ceph-mon[25797]: Detected new or changed devices on smithi189 2024-04-07T14:35:46.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:46 smithi189 ceph-mon[25797]: pgmap v136: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-07T14:35:46.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:46 smithi189 ceph-mon[25797]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-04-07T14:35:46.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:46 smithi189 ceph-mon[25797]: osdmap e42: 6 total, 6 up, 6 in 2024-04-07T14:35:46.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:46 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:46.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:46 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:46.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:46 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:35:46.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:46 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:46.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:46 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:35:46.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:46 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:47.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:47 smithi083 ceph-mon[24810]: pgmap v138: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-07T14:35:47.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:47 smithi083 ceph-mon[24810]: osdmap e43: 6 total, 6 up, 6 in 2024-04-07T14:35:47.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:47 smithi083 ceph-mon[24810]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-04-07T14:35:47.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:47 smithi083 ceph-mon[24810]: from='client.? 172.21.15.189:0/4259952529' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1411a622-2b3f-48f8-b265-8b58643679f0"}]: dispatch 2024-04-07T14:35:47.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:47 smithi083 ceph-mon[24810]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1411a622-2b3f-48f8-b265-8b58643679f0"}]: dispatch 2024-04-07T14:35:47.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:47 smithi083 ceph-mon[24810]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1411a622-2b3f-48f8-b265-8b58643679f0"}]': finished 2024-04-07T14:35:47.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:47 smithi083 ceph-mon[24810]: osdmap e44: 7 total, 6 up, 7 in 2024-04-07T14:35:47.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:47 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-07T14:35:47.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:47 smithi083 ceph-mon[29774]: pgmap v138: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-07T14:35:47.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:47 smithi083 ceph-mon[29774]: osdmap e43: 6 total, 6 up, 6 in 2024-04-07T14:35:47.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:47 smithi083 ceph-mon[29774]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-04-07T14:35:47.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:47 smithi083 ceph-mon[29774]: from='client.? 172.21.15.189:0/4259952529' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1411a622-2b3f-48f8-b265-8b58643679f0"}]: dispatch 2024-04-07T14:35:47.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:47 smithi083 ceph-mon[29774]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1411a622-2b3f-48f8-b265-8b58643679f0"}]: dispatch 2024-04-07T14:35:47.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:47 smithi083 ceph-mon[29774]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1411a622-2b3f-48f8-b265-8b58643679f0"}]': finished 2024-04-07T14:35:47.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:47 smithi083 ceph-mon[29774]: osdmap e44: 7 total, 6 up, 7 in 2024-04-07T14:35:47.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:47 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-07T14:35:47.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:47 smithi189 ceph-mon[25797]: pgmap v138: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-07T14:35:47.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:47 smithi189 ceph-mon[25797]: osdmap e43: 6 total, 6 up, 6 in 2024-04-07T14:35:47.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:47 smithi189 ceph-mon[25797]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-04-07T14:35:47.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:47 smithi189 ceph-mon[25797]: from='client.? 172.21.15.189:0/4259952529' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1411a622-2b3f-48f8-b265-8b58643679f0"}]: dispatch 2024-04-07T14:35:47.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:47 smithi189 ceph-mon[25797]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1411a622-2b3f-48f8-b265-8b58643679f0"}]: dispatch 2024-04-07T14:35:47.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:47 smithi189 ceph-mon[25797]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1411a622-2b3f-48f8-b265-8b58643679f0"}]': finished 2024-04-07T14:35:47.460 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:47 smithi189 ceph-mon[25797]: osdmap e44: 7 total, 6 up, 7 in 2024-04-07T14:35:47.460 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:47 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-07T14:35:48.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:48 smithi083 ceph-mon[24810]: from='client.? 172.21.15.189:0/3323421473' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-07T14:35:48.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:48 smithi083 ceph-mon[29774]: from='client.? 172.21.15.189:0/3323421473' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-07T14:35:48.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:48 smithi189 ceph-mon[25797]: from='client.? 172.21.15.189:0/3323421473' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-07T14:35:49.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:49 smithi189 ceph-mon[25797]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 177 KiB/s, 0 objects/s recovering 2024-04-07T14:35:49.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:49 smithi189 ceph-mon[25797]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-07T14:35:49.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:49 smithi189 ceph-mon[25797]: Cluster is now healthy 2024-04-07T14:35:49.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:49 smithi083 ceph-mon[24810]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 177 KiB/s, 0 objects/s recovering 2024-04-07T14:35:49.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:49 smithi083 ceph-mon[24810]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-07T14:35:49.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:49 smithi083 ceph-mon[24810]: Cluster is now healthy 2024-04-07T14:35:49.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:49 smithi083 ceph-mon[29774]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 177 KiB/s, 0 objects/s recovering 2024-04-07T14:35:49.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:49 smithi083 ceph-mon[29774]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-07T14:35:49.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:49 smithi083 ceph-mon[29774]: Cluster is now healthy 2024-04-07T14:35:51.333 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:51 smithi189 ceph-mon[25797]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 108 KiB/s, 0 objects/s recovering 2024-04-07T14:35:51.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:51 smithi083 ceph-mon[24810]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 108 KiB/s, 0 objects/s recovering 2024-04-07T14:35:51.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:51 smithi083 ceph-mon[29774]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 108 KiB/s, 0 objects/s recovering 2024-04-07T14:35:53.529 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:53 smithi189 ceph-mon[25797]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-04-07T14:35:53.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:53 smithi083 ceph-mon[24810]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-04-07T14:35:53.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:53 smithi083 ceph-mon[29774]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-04-07T14:35:54.481 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:54 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-07T14:35:54.481 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:54 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:54.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:54 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-07T14:35:54.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:54 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:54.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:54 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-07T14:35:54.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:54 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:55.335 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:55 smithi189 ceph-mon[25797]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-07T14:35:55.335 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:55 smithi189 ceph-mon[25797]: Deploying daemon osd.6 on smithi189 2024-04-07T14:35:55.335 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:55 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:55.335 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:55 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:55.425 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:35:55 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:35:55] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:35:55.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:55 smithi083 ceph-mon[24810]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-07T14:35:55.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:55 smithi083 ceph-mon[24810]: Deploying daemon osd.6 on smithi189 2024-04-07T14:35:55.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:55 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:55.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:55 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:55.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:55 smithi083 ceph-mon[29774]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-07T14:35:55.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:55 smithi083 ceph-mon[29774]: Deploying daemon osd.6 on smithi189 2024-04-07T14:35:55.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:55 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:55.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:55 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:57.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:57 smithi189 ceph-mon[25797]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 55 KiB/s, 0 objects/s recovering 2024-04-07T14:35:57.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:57 smithi083 ceph-mon[24810]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 55 KiB/s, 0 objects/s recovering 2024-04-07T14:35:57.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:57 smithi083 ceph-mon[29774]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 55 KiB/s, 0 objects/s recovering 2024-04-07T14:35:59.291 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:59 smithi189 ceph-mon[25797]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-04-07T14:35:59.291 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:59 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:59.291 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:59 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:35:59.291 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:59 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:59.291 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:59 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:35:59.291 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:35:59 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:59.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:59 smithi083 ceph-mon[24810]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-04-07T14:35:59.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:59 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:59.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:59 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:35:59.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:59 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:59.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:59 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:35:59.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:35:59 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:59.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:59 smithi083 ceph-mon[29774]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-04-07T14:35:59.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:59 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:35:59.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:59 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:35:59.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:59 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:35:59.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:59 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:35:59.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:35:59 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:01.334 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:01 smithi189 ceph-mon[25797]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-07T14:36:01.366 INFO:teuthology.orchestra.run.smithi189.stdout:Created osd(s) 6 on host 'smithi189' 2024-04-07T14:36:01.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:01 smithi083 ceph-mon[24810]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-07T14:36:01.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:01 smithi083 ceph-mon[29774]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-07T14:36:01.885 DEBUG:teuthology.orchestra.run.smithi189:osd.6> sudo journalctl -f -n 0 -u ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@osd.6.service 2024-04-07T14:36:01.887 INFO:tasks.cephadm:Deploying osd.7 on smithi189 with /dev/vg_nvme/lv_1... 2024-04-07T14:36:01.888 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-07T14:36:02.532 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:02 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:02.533 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:02 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:36:02.533 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:02 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:36:02.533 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:02 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:36:02.533 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:02 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:36:02.533 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:02 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:36:02.533 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:02 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:36:02.533 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:02 smithi189 ceph-mon[25797]: Adjusting osd_memory_target on smithi189 to 5354M 2024-04-07T14:36:02.533 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:02 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:02.533 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:02 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:02.533 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:02 smithi189 ceph-mon[25797]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-07T14:36:02.533 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:02 smithi189 ceph-mon[25797]: from='osd.6 [v2:172.21.15.189:6816/3839485321,v1:172.21.15.189:6817/3839485321]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-07T14:36:02.533 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:02 smithi189 ceph-mon[25797]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-07T14:36:02.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:02 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:02.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:02 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:36:02.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:02 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:36:02.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:02 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:36:02.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:02 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:36:02.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:02 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:36:02.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:02 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:36:02.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:02 smithi083 ceph-mon[24810]: Adjusting osd_memory_target on smithi189 to 5354M 2024-04-07T14:36:02.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:02 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:02.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:02 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:02.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:02 smithi083 ceph-mon[24810]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-07T14:36:02.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:02 smithi083 ceph-mon[24810]: from='osd.6 [v2:172.21.15.189:6816/3839485321,v1:172.21.15.189:6817/3839485321]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-07T14:36:02.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:02 smithi083 ceph-mon[24810]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-07T14:36:02.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:02 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:02.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:02 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:36:02.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:02 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:36:02.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:02 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:36:02.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:02 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:36:02.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:02 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:36:02.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:02 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:36:02.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:02 smithi083 ceph-mon[29774]: Adjusting osd_memory_target on smithi189 to 5354M 2024-04-07T14:36:02.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:02 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:02.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:02 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:02.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:02 smithi083 ceph-mon[29774]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-07T14:36:02.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:02 smithi083 ceph-mon[29774]: from='osd.6 [v2:172.21.15.189:6816/3839485321,v1:172.21.15.189:6817/3839485321]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-07T14:36:02.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:02 smithi083 ceph-mon[29774]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-07T14:36:03.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:03 smithi189 ceph-mon[25797]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-07T14:36:03.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:03 smithi189 ceph-mon[25797]: osdmap e45: 7 total, 6 up, 7 in 2024-04-07T14:36:03.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:03 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-07T14:36:03.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:03 smithi189 ceph-mon[25797]: from='osd.6 [v2:172.21.15.189:6816/3839485321,v1:172.21.15.189:6817/3839485321]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-04-07T14:36:03.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:03 smithi189 ceph-mon[25797]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-04-07T14:36:03.709 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:36:03 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:36:03.437+0000 7efca87f3700 -1 osd.6 0 waiting for initial osdmap 2024-04-07T14:36:03.709 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:36:03 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:36:03.445+0000 7efca418c700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-07T14:36:03.873 INFO:teuthology.orchestra.run.smithi189.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-07T14:36:03.873 INFO:teuthology.orchestra.run.smithi189.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-07T14:36:03.873 INFO:teuthology.orchestra.run.smithi189.stderr: stderr: 10+0 records in 2024-04-07T14:36:03.873 INFO:teuthology.orchestra.run.smithi189.stderr:10+0 records out 2024-04-07T14:36:03.873 INFO:teuthology.orchestra.run.smithi189.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0124179 s, 844 MB/s 2024-04-07T14:36:03.873 INFO:teuthology.orchestra.run.smithi189.stderr:--> Zapping successful for: 2024-04-07T14:36:03.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:03 smithi083 ceph-mon[24810]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-07T14:36:03.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:03 smithi083 ceph-mon[24810]: osdmap e45: 7 total, 6 up, 7 in 2024-04-07T14:36:03.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:03 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-07T14:36:03.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:03 smithi083 ceph-mon[24810]: from='osd.6 [v2:172.21.15.189:6816/3839485321,v1:172.21.15.189:6817/3839485321]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-04-07T14:36:03.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:03 smithi083 ceph-mon[24810]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-04-07T14:36:03.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:03 smithi083 ceph-mon[29774]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-07T14:36:03.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:03 smithi083 ceph-mon[29774]: osdmap e45: 7 total, 6 up, 7 in 2024-04-07T14:36:03.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:03 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-07T14:36:03.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:03 smithi083 ceph-mon[29774]: from='osd.6 [v2:172.21.15.189:6816/3839485321,v1:172.21.15.189:6817/3839485321]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-04-07T14:36:03.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:03 smithi083 ceph-mon[29774]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-04-07T14:36:04.536 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph orch daemon add osd smithi189:vg_nvme/lv_1 2024-04-07T14:36:04.563 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:04 smithi189 ceph-mon[25797]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-04-07T14:36:04.564 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:04 smithi189 ceph-mon[25797]: osdmap e46: 7 total, 6 up, 7 in 2024-04-07T14:36:04.564 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:04 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-07T14:36:04.564 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:04 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-07T14:36:04.564 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:04 smithi189 ceph-mon[25797]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-07T14:36:04.564 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:04 smithi189 ceph-mon[25797]: osd.6 [v2:172.21.15.189:6816/3839485321,v1:172.21.15.189:6817/3839485321] boot 2024-04-07T14:36:04.564 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:04 smithi189 ceph-mon[25797]: osdmap e47: 7 total, 7 up, 7 in 2024-04-07T14:36:04.564 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:04 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-07T14:36:04.827 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:04 smithi083 ceph-mon[29774]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-04-07T14:36:04.827 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:04 smithi083 ceph-mon[29774]: osdmap e46: 7 total, 6 up, 7 in 2024-04-07T14:36:04.828 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:04 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-07T14:36:04.828 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:04 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-07T14:36:04.828 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:04 smithi083 ceph-mon[29774]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-07T14:36:04.828 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:04 smithi083 ceph-mon[29774]: osd.6 [v2:172.21.15.189:6816/3839485321,v1:172.21.15.189:6817/3839485321] boot 2024-04-07T14:36:04.828 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:04 smithi083 ceph-mon[29774]: osdmap e47: 7 total, 7 up, 7 in 2024-04-07T14:36:04.828 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:04 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-07T14:36:04.828 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:04 smithi083 ceph-mon[24810]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-04-07T14:36:04.828 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:04 smithi083 ceph-mon[24810]: osdmap e46: 7 total, 6 up, 7 in 2024-04-07T14:36:04.828 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:04 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-07T14:36:04.828 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:04 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-07T14:36:04.828 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:04 smithi083 ceph-mon[24810]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-07T14:36:04.829 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:04 smithi083 ceph-mon[24810]: osd.6 [v2:172.21.15.189:6816/3839485321,v1:172.21.15.189:6817/3839485321] boot 2024-04-07T14:36:04.829 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:04 smithi083 ceph-mon[24810]: osdmap e47: 7 total, 7 up, 7 in 2024-04-07T14:36:04.829 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:04 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-07T14:36:05.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:05 smithi189 ceph-mon[25797]: purged_snaps scrub starts 2024-04-07T14:36:05.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:05 smithi189 ceph-mon[25797]: purged_snaps scrub ok 2024-04-07T14:36:05.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:05 smithi189 ceph-mon[25797]: osdmap e48: 7 total, 7 up, 7 in 2024-04-07T14:36:05.926 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:05 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:36:05] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:36:05.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:05 smithi083 ceph-mon[29774]: purged_snaps scrub starts 2024-04-07T14:36:05.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:05 smithi083 ceph-mon[29774]: purged_snaps scrub ok 2024-04-07T14:36:05.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:05 smithi083 ceph-mon[29774]: osdmap e48: 7 total, 7 up, 7 in 2024-04-07T14:36:05.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:05 smithi083 ceph-mon[24810]: purged_snaps scrub starts 2024-04-07T14:36:05.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:05 smithi083 ceph-mon[24810]: purged_snaps scrub ok 2024-04-07T14:36:05.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:05 smithi083 ceph-mon[24810]: osdmap e48: 7 total, 7 up, 7 in 2024-04-07T14:36:06.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:06 smithi189 ceph-mon[25797]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-07T14:36:06.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:06 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:06.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:06 smithi189 ceph-mon[25797]: Detected new or changed devices on smithi189 2024-04-07T14:36:06.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:06 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:06.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:06 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:36:06.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:06 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:36:06.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:06 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:36:06.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:06 smithi189 ceph-mon[25797]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-07T14:36:06.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:06 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:06.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:06 smithi189 ceph-mon[25797]: osdmap e49: 7 total, 7 up, 7 in 2024-04-07T14:36:06.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:06 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-07T14:36:06.710 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:06 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-07T14:36:06.710 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:06 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:36:06.710 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:06 smithi189 ceph-mon[25797]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-04-07T14:36:06.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:06 smithi083 ceph-mon[24810]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-07T14:36:06.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:06 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:06.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:06 smithi083 ceph-mon[24810]: Detected new or changed devices on smithi189 2024-04-07T14:36:06.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:06 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:06.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:06 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:36:06.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:06 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:36:06.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:06 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:36:06.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:06 smithi083 ceph-mon[24810]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-07T14:36:06.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:06 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:06.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:06 smithi083 ceph-mon[24810]: osdmap e49: 7 total, 7 up, 7 in 2024-04-07T14:36:06.928 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:06 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-07T14:36:06.928 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:06 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-07T14:36:06.928 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:06 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:36:06.928 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:06 smithi083 ceph-mon[24810]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-04-07T14:36:06.928 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:06 smithi083 ceph-mon[29774]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-07T14:36:06.928 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:06 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:06.929 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:06 smithi083 ceph-mon[29774]: Detected new or changed devices on smithi189 2024-04-07T14:36:06.929 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:06 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:06.929 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:06 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:36:06.929 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:06 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:36:06.929 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:06 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:36:06.929 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:06 smithi083 ceph-mon[29774]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-07T14:36:06.929 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:06 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:06.929 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:06 smithi083 ceph-mon[29774]: osdmap e49: 7 total, 7 up, 7 in 2024-04-07T14:36:06.930 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:06 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-07T14:36:06.930 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:06 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-07T14:36:06.930 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:06 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:36:06.930 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:06 smithi083 ceph-mon[29774]: Health check failed: 1 stray daemon(s) not managed by cephadm (CEPHADM_STRAY_DAEMON) 2024-04-07T14:36:07.813 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:07 smithi189 ceph-mon[25797]: from='client.24278 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:36:07.813 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:07 smithi189 ceph-mon[25797]: osdmap e50: 7 total, 7 up, 7 in 2024-04-07T14:36:07.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:07 smithi083 ceph-mon[24810]: from='client.24278 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:36:07.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:07 smithi083 ceph-mon[24810]: osdmap e50: 7 total, 7 up, 7 in 2024-04-07T14:36:07.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:07 smithi083 ceph-mon[29774]: from='client.24278 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi189:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:36:07.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:07 smithi083 ceph-mon[29774]: osdmap e50: 7 total, 7 up, 7 in 2024-04-07T14:36:08.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:08 smithi189 ceph-mon[25797]: pgmap v158: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail; 158 KiB/s, 0 objects/s recovering 2024-04-07T14:36:08.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:08 smithi189 ceph-mon[25797]: osdmap e51: 7 total, 7 up, 7 in 2024-04-07T14:36:08.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:08 smithi083 ceph-mon[24810]: pgmap v158: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail; 158 KiB/s, 0 objects/s recovering 2024-04-07T14:36:08.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:08 smithi083 ceph-mon[24810]: osdmap e51: 7 total, 7 up, 7 in 2024-04-07T14:36:08.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:08 smithi083 ceph-mon[29774]: pgmap v158: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail; 158 KiB/s, 0 objects/s recovering 2024-04-07T14:36:08.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:08 smithi083 ceph-mon[29774]: osdmap e51: 7 total, 7 up, 7 in 2024-04-07T14:36:09.676 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:09 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:36:09] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:36:10.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:10 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:10.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:10 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:10.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:10 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:36:10.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:10 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:36:10.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:10 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:36:10.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:10 smithi189 ceph-mon[25797]: pgmap v160: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail; 141 KiB/s, 0 objects/s recovering 2024-04-07T14:36:10.460 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:10 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:10.460 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:10 smithi189 ceph-mon[25797]: from='client.? 172.21.15.189:0/2882480329' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bbaaa4cf-7ff3-4a75-aeb8-d8325b9313c1"}]: dispatch 2024-04-07T14:36:10.460 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:10 smithi189 ceph-mon[25797]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bbaaa4cf-7ff3-4a75-aeb8-d8325b9313c1"}]: dispatch 2024-04-07T14:36:10.460 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:10 smithi189 ceph-mon[25797]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bbaaa4cf-7ff3-4a75-aeb8-d8325b9313c1"}]': finished 2024-04-07T14:36:10.460 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:10 smithi189 ceph-mon[25797]: osdmap e52: 8 total, 7 up, 8 in 2024-04-07T14:36:10.460 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:10 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-07T14:36:10.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:10 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:10.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:10 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:10.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:10 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:36:10.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:10 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:36:10.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:10 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:36:10.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:10 smithi083 ceph-mon[24810]: pgmap v160: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail; 141 KiB/s, 0 objects/s recovering 2024-04-07T14:36:10.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:10 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:10.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:10 smithi083 ceph-mon[24810]: from='client.? 172.21.15.189:0/2882480329' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bbaaa4cf-7ff3-4a75-aeb8-d8325b9313c1"}]: dispatch 2024-04-07T14:36:10.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:10 smithi083 ceph-mon[24810]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bbaaa4cf-7ff3-4a75-aeb8-d8325b9313c1"}]: dispatch 2024-04-07T14:36:10.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:10 smithi083 ceph-mon[24810]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bbaaa4cf-7ff3-4a75-aeb8-d8325b9313c1"}]': finished 2024-04-07T14:36:10.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:10 smithi083 ceph-mon[24810]: osdmap e52: 8 total, 7 up, 8 in 2024-04-07T14:36:10.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:10 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-07T14:36:10.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:10 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:10.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:10 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:10.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:10 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:36:10.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:10 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:36:10.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:10 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:36:10.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:10 smithi083 ceph-mon[29774]: pgmap v160: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail; 141 KiB/s, 0 objects/s recovering 2024-04-07T14:36:10.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:10 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:10.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:10 smithi083 ceph-mon[29774]: from='client.? 172.21.15.189:0/2882480329' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bbaaa4cf-7ff3-4a75-aeb8-d8325b9313c1"}]: dispatch 2024-04-07T14:36:10.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:10 smithi083 ceph-mon[29774]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bbaaa4cf-7ff3-4a75-aeb8-d8325b9313c1"}]: dispatch 2024-04-07T14:36:10.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:10 smithi083 ceph-mon[29774]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bbaaa4cf-7ff3-4a75-aeb8-d8325b9313c1"}]': finished 2024-04-07T14:36:10.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:10 smithi083 ceph-mon[29774]: osdmap e52: 8 total, 7 up, 8 in 2024-04-07T14:36:10.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:10 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-07T14:36:11.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:11 smithi189 ceph-mon[25797]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-04-07T14:36:11.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:11 smithi189 ceph-mon[25797]: Cluster is now healthy 2024-04-07T14:36:11.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:11 smithi189 ceph-mon[25797]: from='client.? 172.21.15.189:0/2301995498' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-07T14:36:11.679 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:11 smithi083 ceph-mon[24810]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-04-07T14:36:11.679 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:11 smithi083 ceph-mon[24810]: Cluster is now healthy 2024-04-07T14:36:11.679 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:11 smithi083 ceph-mon[24810]: from='client.? 172.21.15.189:0/2301995498' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-07T14:36:11.679 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:11 smithi083 ceph-mon[29774]: Health check cleared: CEPHADM_STRAY_DAEMON (was: 1 stray daemon(s) not managed by cephadm) 2024-04-07T14:36:11.679 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:11 smithi083 ceph-mon[29774]: Cluster is now healthy 2024-04-07T14:36:11.679 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:11 smithi083 ceph-mon[29774]: from='client.? 172.21.15.189:0/2301995498' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-07T14:36:12.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:12 smithi189 ceph-mon[25797]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 103 KiB/s, 0 objects/s recovering 2024-04-07T14:36:12.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:12 smithi083 ceph-mon[24810]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 103 KiB/s, 0 objects/s recovering 2024-04-07T14:36:12.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:12 smithi083 ceph-mon[29774]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 103 KiB/s, 0 objects/s recovering 2024-04-07T14:36:13.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:13 smithi189 ceph-mon[25797]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-07T14:36:13.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:13 smithi083 ceph-mon[24810]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-07T14:36:13.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:13 smithi083 ceph-mon[29774]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-07T14:36:15.676 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:15 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:36:15] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:36:16.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:16 smithi083 ceph-mon[24810]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-07T14:36:16.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:16 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:16.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:16 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:16.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:16 smithi083 ceph-mon[29774]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-07T14:36:16.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:16 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:16.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:16 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:16.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:16 smithi189 ceph-mon[25797]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-07T14:36:16.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:16 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:16.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:16 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:18.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:18 smithi189 ceph-mon[25797]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-07T14:36:18.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:18 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-07T14:36:18.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:18 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:36:18.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:18 smithi083 ceph-mon[24810]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-07T14:36:18.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:18 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-07T14:36:18.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:18 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:36:18.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:18 smithi083 ceph-mon[29774]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-07T14:36:18.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:18 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-07T14:36:18.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:18 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:36:19.373 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:19 smithi189 ceph-mon[25797]: Deploying daemon osd.7 on smithi189 2024-04-07T14:36:19.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:19 smithi083 ceph-mon[24810]: Deploying daemon osd.7 on smithi189 2024-04-07T14:36:19.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:19 smithi083 ceph-mon[29774]: Deploying daemon osd.7 on smithi189 2024-04-07T14:36:20.356 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:20 smithi189 ceph-mon[25797]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-07T14:36:20.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:20 smithi083 ceph-mon[24810]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-07T14:36:20.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:20 smithi083 ceph-mon[29774]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-07T14:36:22.391 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:22 smithi189 ceph-mon[25797]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-07T14:36:22.392 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:22 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:36:22.392 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:22 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:36:22.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:22 smithi083 ceph-mon[24810]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-07T14:36:22.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:22 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:36:22.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:22 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:36:22.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:22 smithi083 ceph-mon[29774]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-07T14:36:22.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:22 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:36:22.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:22 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:36:23.673 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:23 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:23.673 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:23 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:36:23.673 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:23 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:36:23.673 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:23 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:36:23.674 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:23 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:23.674 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:23 smithi189 ceph-mon[25797]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-07T14:36:23.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:23 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:23.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:23 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:36:23.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:23 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:36:23.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:23 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:36:23.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:23 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:23.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:23 smithi083 ceph-mon[29774]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-07T14:36:23.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:23 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:23.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:23 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:36:23.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:23 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:36:23.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:23 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:36:23.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:23 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:23.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:23 smithi083 ceph-mon[24810]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-07T14:36:25.239 INFO:teuthology.orchestra.run.smithi189.stdout:Created osd(s) 7 on host 'smithi189' 2024-04-07T14:36:25.884 DEBUG:teuthology.orchestra.run.smithi189:osd.7> sudo journalctl -f -n 0 -u ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@osd.7.service 2024-04-07T14:36:25.927 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-04-07T14:36:25.927 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph osd stat -f json 2024-04-07T14:36:26.490 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:26 smithi083 ceph-mon[24810]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-07T14:36:26.490 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:26 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:26.490 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:26 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:36:26.490 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:26 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:36:26.490 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:26 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:36:26.490 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:26 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:36:26.490 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:26 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:36:26.491 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:26 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:36:26.491 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:26 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:36:26.491 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:26 smithi083 ceph-mon[24810]: Adjusting osd_memory_target on smithi189 to 4016M 2024-04-07T14:36:26.491 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:26 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:26.491 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:26 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:26.491 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:26 smithi083 ceph-mon[24810]: from='osd.7 [v2:172.21.15.189:6824/764170876,v1:172.21.15.189:6825/764170876]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-07T14:36:26.491 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:26 smithi083 ceph-mon[24810]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-07T14:36:26.491 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:26 smithi083 ceph-mon[29774]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-07T14:36:26.491 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:26 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:26.491 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:26 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:36:26.491 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:26 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:36:26.491 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:26 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:36:26.492 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:26 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:36:26.492 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:26 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:36:26.492 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:26 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:36:26.492 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:26 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:36:26.492 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:26 smithi083 ceph-mon[29774]: Adjusting osd_memory_target on smithi189 to 4016M 2024-04-07T14:36:26.492 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:26 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:26.492 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:26 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:26.492 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:26 smithi083 ceph-mon[29774]: from='osd.7 [v2:172.21.15.189:6824/764170876,v1:172.21.15.189:6825/764170876]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-07T14:36:26.492 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:26 smithi083 ceph-mon[29774]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-07T14:36:26.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:26 smithi189 ceph-mon[25797]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-07T14:36:26.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:26 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:26.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:26 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:36:26.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:26 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:36:26.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:26 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:36:26.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:26 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:36:26.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:26 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:36:26.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:26 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:36:26.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:26 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:36:26.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:26 smithi189 ceph-mon[25797]: Adjusting osd_memory_target on smithi189 to 4016M 2024-04-07T14:36:26.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:26 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:26.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:26 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:26.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:26 smithi189 ceph-mon[25797]: from='osd.7 [v2:172.21.15.189:6824/764170876,v1:172.21.15.189:6825/764170876]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-07T14:36:26.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:26 smithi189 ceph-mon[25797]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-07T14:36:27.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:27 smithi189 ceph-mon[25797]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-07T14:36:27.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:27 smithi189 ceph-mon[25797]: osdmap e53: 8 total, 7 up, 8 in 2024-04-07T14:36:27.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:27 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-07T14:36:27.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:27 smithi189 ceph-mon[25797]: from='osd.7 [v2:172.21.15.189:6824/764170876,v1:172.21.15.189:6825/764170876]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-04-07T14:36:27.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:27 smithi189 ceph-mon[25797]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-04-07T14:36:27.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:27 smithi189 ceph-mon[25797]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-07T14:36:27.459 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:36:27 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:36:27.293+0000 7f7c97a50700 -1 osd.7 0 waiting for initial osdmap 2024-04-07T14:36:27.459 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:36:27 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:36:27.302+0000 7f7c92be8700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-07T14:36:27.544 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:27 smithi083 ceph-mon[29774]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-07T14:36:27.544 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:27 smithi083 ceph-mon[29774]: osdmap e53: 8 total, 7 up, 8 in 2024-04-07T14:36:27.544 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:27 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-07T14:36:27.544 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:27 smithi083 ceph-mon[29774]: from='osd.7 [v2:172.21.15.189:6824/764170876,v1:172.21.15.189:6825/764170876]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-04-07T14:36:27.544 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:27 smithi083 ceph-mon[29774]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-04-07T14:36:27.544 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:27 smithi083 ceph-mon[29774]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-07T14:36:27.545 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:27 smithi083 ceph-mon[24810]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-07T14:36:27.545 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:27 smithi083 ceph-mon[24810]: osdmap e53: 8 total, 7 up, 8 in 2024-04-07T14:36:27.545 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:27 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-07T14:36:27.545 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:27 smithi083 ceph-mon[24810]: from='osd.7 [v2:172.21.15.189:6824/764170876,v1:172.21.15.189:6825/764170876]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-04-07T14:36:27.545 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:27 smithi083 ceph-mon[24810]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]: dispatch 2024-04-07T14:36:27.545 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:27 smithi083 ceph-mon[24810]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-07T14:36:28.083 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-07T14:36:28.343 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:28 smithi083 ceph-mon[29774]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-04-07T14:36:28.343 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:28 smithi083 ceph-mon[29774]: osdmap e54: 8 total, 7 up, 8 in 2024-04-07T14:36:28.343 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:28 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-07T14:36:28.343 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:28 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-07T14:36:28.344 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:28 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/1840238900' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-07T14:36:28.344 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:28 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-07T14:36:28.344 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:28 smithi083 ceph-mon[24810]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-04-07T14:36:28.344 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:28 smithi083 ceph-mon[24810]: osdmap e54: 8 total, 7 up, 8 in 2024-04-07T14:36:28.344 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:28 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-07T14:36:28.344 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:28 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-07T14:36:28.344 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:28 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/1840238900' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-07T14:36:28.344 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:28 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-07T14:36:28.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:28 smithi189 ceph-mon[25797]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi189", "root=default"]}]': finished 2024-04-07T14:36:28.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:28 smithi189 ceph-mon[25797]: osdmap e54: 8 total, 7 up, 8 in 2024-04-07T14:36:28.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:28 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-07T14:36:28.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:28 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-07T14:36:28.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:28 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/1840238900' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-07T14:36:28.584 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:28 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-07T14:36:28.675 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:28 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:36:28] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:36:28.762 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":54,"num_osds":8,"num_up_osds":7,"osd_up_since":1712500564,"num_in_osds":8,"osd_in_since":1712500570,"num_remapped_pgs":0} 2024-04-07T14:36:29.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:29 smithi083 ceph-mon[24810]: purged_snaps scrub starts 2024-04-07T14:36:29.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:29 smithi083 ceph-mon[24810]: purged_snaps scrub ok 2024-04-07T14:36:29.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:29 smithi083 ceph-mon[24810]: osd.7 [v2:172.21.15.189:6824/764170876,v1:172.21.15.189:6825/764170876] boot 2024-04-07T14:36:29.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:29 smithi083 ceph-mon[24810]: osdmap e55: 8 total, 8 up, 8 in 2024-04-07T14:36:29.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:29 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-07T14:36:29.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:29 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:29.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:29 smithi083 ceph-mon[24810]: Detected new or changed devices on smithi189 2024-04-07T14:36:29.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:29 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:29.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:29 smithi083 ceph-mon[24810]: pgmap v174: 1 pgs: 1 peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-07T14:36:29.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:29 smithi083 ceph-mon[29774]: purged_snaps scrub starts 2024-04-07T14:36:29.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:29 smithi083 ceph-mon[29774]: purged_snaps scrub ok 2024-04-07T14:36:29.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:29 smithi083 ceph-mon[29774]: osd.7 [v2:172.21.15.189:6824/764170876,v1:172.21.15.189:6825/764170876] boot 2024-04-07T14:36:29.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:29 smithi083 ceph-mon[29774]: osdmap e55: 8 total, 8 up, 8 in 2024-04-07T14:36:29.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:29 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-07T14:36:29.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:29 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:29.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:29 smithi083 ceph-mon[29774]: Detected new or changed devices on smithi189 2024-04-07T14:36:29.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:29 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:29.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:29 smithi083 ceph-mon[29774]: pgmap v174: 1 pgs: 1 peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-07T14:36:29.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:29 smithi189 ceph-mon[25797]: purged_snaps scrub starts 2024-04-07T14:36:29.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:29 smithi189 ceph-mon[25797]: purged_snaps scrub ok 2024-04-07T14:36:29.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:29 smithi189 ceph-mon[25797]: osd.7 [v2:172.21.15.189:6824/764170876,v1:172.21.15.189:6825/764170876] boot 2024-04-07T14:36:29.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:29 smithi189 ceph-mon[25797]: osdmap e55: 8 total, 8 up, 8 in 2024-04-07T14:36:29.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:29 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-07T14:36:29.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:29 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:29.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:29 smithi189 ceph-mon[25797]: Detected new or changed devices on smithi189 2024-04-07T14:36:29.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:29 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:29.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:29 smithi189 ceph-mon[25797]: pgmap v174: 1 pgs: 1 peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-07T14:36:29.763 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph osd stat -f json 2024-04-07T14:36:30.335 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:30 smithi083 ceph-mon[24810]: osdmap e56: 8 total, 8 up, 8 in 2024-04-07T14:36:30.335 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:30 smithi083 ceph-mon[24810]: osdmap e57: 8 total, 8 up, 8 in 2024-04-07T14:36:30.335 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:30 smithi083 ceph-mon[29774]: osdmap e56: 8 total, 8 up, 8 in 2024-04-07T14:36:30.335 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:30 smithi083 ceph-mon[29774]: osdmap e57: 8 total, 8 up, 8 in 2024-04-07T14:36:30.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:30 smithi189 ceph-mon[25797]: osdmap e56: 8 total, 8 up, 8 in 2024-04-07T14:36:30.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:30 smithi189 ceph-mon[25797]: osdmap e57: 8 total, 8 up, 8 in 2024-04-07T14:36:31.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:31 smithi083 ceph-mon[24810]: pgmap v177: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:36:31.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:31 smithi083 ceph-mon[29774]: pgmap v177: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:36:31.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:31 smithi189 ceph-mon[25797]: pgmap v177: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:36:31.858 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-07T14:36:32.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:32 smithi083 ceph-mon[29774]: osdmap e58: 8 total, 8 up, 8 in 2024-04-07T14:36:32.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:32 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/2473778711' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-07T14:36:32.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:32 smithi083 ceph-mon[24810]: osdmap e58: 8 total, 8 up, 8 in 2024-04-07T14:36:32.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:32 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/2473778711' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-07T14:36:32.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:32 smithi189 ceph-mon[25797]: osdmap e58: 8 total, 8 up, 8 in 2024-04-07T14:36:32.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:32 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/2473778711' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-07T14:36:32.539 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":58,"num_osds":8,"num_up_osds":8,"osd_up_since":1712500588,"num_in_osds":8,"osd_in_since":1712500570,"num_remapped_pgs":0} 2024-04-07T14:36:32.539 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-07T14:36:32.752 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-07T14:36:33.390 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:33 smithi189 ceph-mon[25797]: osdmap e59: 8 total, 8 up, 8 in 2024-04-07T14:36:33.390 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:33 smithi189 ceph-mon[25797]: pgmap v180: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:36:33.562 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:33 smithi083 ceph-mon[24810]: osdmap e59: 8 total, 8 up, 8 in 2024-04-07T14:36:33.562 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:33 smithi083 ceph-mon[24810]: pgmap v180: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:36:33.562 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:33 smithi083 ceph-mon[29774]: osdmap e59: 8 total, 8 up, 8 in 2024-04-07T14:36:33.562 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:33 smithi083 ceph-mon[29774]: pgmap v180: 1 pgs: 1 peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:36:34.577 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-07T14:36:34.577 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":59,"fsid":"7388281a-f4eb-11ee-b649-cb9ed24678a4","created":"2024-04-07T14:31:30.809811+0000","modified":"2024-04-07T14:36:32.311792+0000","last_up_change":"2024-04-07T14:36:28.289077+0000","last_in_change":"2024-04-07T14:36:10.041386+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-07T14:34:36.411287+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"a42cc075-c4af-4500-abae-738cfd1c8d97","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":159265472},{"type":"v1","addr":"172.21.15.83:6803","nonce":159265472}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":159265472},{"type":"v1","addr":"172.21.15.83:6805","nonce":159265472}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":159265472},{"type":"v1","addr":"172.21.15.83:6809","nonce":159265472}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":159265472},{"type":"v1","addr":"172.21.15.83:6807","nonce":159265472}]},"public_addr":"172.21.15.83:6803/159265472","cluster_addr":"172.21.15.83:6805/159265472","heartbeat_back_addr":"172.21.15.83:6809/159265472","heartbeat_front_addr":"172.21.15.83:6807/159265472","state":["exists","up"]},{"osd":1,"uuid":"c0b76e30-ff29-43b7-a8d1-90f9bef15b7e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6810","nonce":4165704838},{"type":"v1","addr":"172.21.15.83:6811","nonce":4165704838}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6812","nonce":4165704838},{"type":"v1","addr":"172.21.15.83:6813","nonce":4165704838}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6816","nonce":4165704838},{"type":"v1","addr":"172.21.15.83:6817","nonce":4165704838}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6814","nonce":4165704838},{"type":"v1","addr":"172.21.15.83:6815","nonce":4165704838}]},"public_addr":"172.21.15.83:6811/4165704838","cluster_addr":"172.21.15.83:6813/4165704838","heartbeat_back_addr":"172.21.15.83:6817/4165704838","heartbeat_front_addr":"172.21.15.83:6815/4165704838","state":["exists","up"]},{"osd":2,"uuid":"fe1ee70e-866b-463e-b7fd-2a2057f44b80","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6818","nonce":718005878},{"type":"v1","addr":"172.21.15.83:6819","nonce":718005878}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6820","nonce":718005878},{"type":"v1","addr":"172.21.15.83:6821","nonce":718005878}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6824","nonce":718005878},{"type":"v1","addr":"172.21.15.83:6825","nonce":718005878}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6822","nonce":718005878},{"type":"v1","addr":"172.21.15.83:6823","nonce":718005878}]},"public_addr":"172.21.15.83:6819/718005878","cluster_addr":"172.21.15.83:6821/718005878","heartbeat_back_addr":"172.21.15.83:6825/718005878","heartbeat_front_addr":"172.21.15.83:6823/718005878","state":["exists","up"]},{"osd":3,"uuid":"14bfed5e-6a77-4448-855b-258e622311ee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6826","nonce":3079630945},{"type":"v1","addr":"172.21.15.83:6827","nonce":3079630945}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6828","nonce":3079630945},{"type":"v1","addr":"172.21.15.83:6829","nonce":3079630945}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6832","nonce":3079630945},{"type":"v1","addr":"172.21.15.83:6833","nonce":3079630945}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6830","nonce":3079630945},{"type":"v1","addr":"172.21.15.83:6831","nonce":3079630945}]},"public_addr":"172.21.15.83:6827/3079630945","cluster_addr":"172.21.15.83:6829/3079630945","heartbeat_back_addr":"172.21.15.83:6833/3079630945","heartbeat_front_addr":"172.21.15.83:6831/3079630945","state":["exists","up"]},{"osd":4,"uuid":"6a8687be-8c65-40a3-89c4-d463ed60d920","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6800","nonce":3355623828},{"type":"v1","addr":"172.21.15.189:6801","nonce":3355623828}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6802","nonce":3355623828},{"type":"v1","addr":"172.21.15.189:6803","nonce":3355623828}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6806","nonce":3355623828},{"type":"v1","addr":"172.21.15.189:6807","nonce":3355623828}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6804","nonce":3355623828},{"type":"v1","addr":"172.21.15.189:6805","nonce":3355623828}]},"public_addr":"172.21.15.189:6801/3355623828","cluster_addr":"172.21.15.189:6803/3355623828","heartbeat_back_addr":"172.21.15.189:6807/3355623828","heartbeat_front_addr":"172.21.15.189:6805/3355623828","state":["exists","up"]},{"osd":5,"uuid":"7115ab07-1339-4eac-b403-bf6b40361af9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6808","nonce":3207731440},{"type":"v1","addr":"172.21.15.189:6809","nonce":3207731440}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6810","nonce":3207731440},{"type":"v1","addr":"172.21.15.189:6811","nonce":3207731440}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6814","nonce":3207731440},{"type":"v1","addr":"172.21.15.189:6815","nonce":3207731440}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6812","nonce":3207731440},{"type":"v1","addr":"172.21.15.189:6813","nonce":3207731440}]},"public_addr":"172.21.15.189:6809/3207731440","cluster_addr":"172.21.15.189:6811/3207731440","heartbeat_back_addr":"172.21.15.189:6815/3207731440","heartbeat_front_addr":"172.21.15.189:6813/3207731440","state":["exists","up"]},{"osd":6,"uuid":"1411a622-2b3f-48f8-b265-8b58643679f0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6816","nonce":3839485321},{"type":"v1","addr":"172.21.15.189:6817","nonce":3839485321}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6818","nonce":3839485321},{"type":"v1","addr":"172.21.15.189:6819","nonce":3839485321}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6822","nonce":3839485321},{"type":"v1","addr":"172.21.15.189:6823","nonce":3839485321}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6820","nonce":3839485321},{"type":"v1","addr":"172.21.15.189:6821","nonce":3839485321}]},"public_addr":"172.21.15.189:6817/3839485321","cluster_addr":"172.21.15.189:6819/3839485321","heartbeat_back_addr":"172.21.15.189:6823/3839485321","heartbeat_front_addr":"172.21.15.189:6821/3839485321","state":["exists","up"]},{"osd":7,"uuid":"bbaaa4cf-7ff3-4a75-aeb8-d8325b9313c1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6824","nonce":764170876},{"type":"v1","addr":"172.21.15.189:6825","nonce":764170876}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6826","nonce":764170876},{"type":"v1","addr":"172.21.15.189:6827","nonce":764170876}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6830","nonce":764170876},{"type":"v1","addr":"172.21.15.189:6831","nonce":764170876}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6828","nonce":764170876},{"type":"v1","addr":"172.21.15.189:6829","nonce":764170876}]},"public_addr":"172.21.15.189:6825/764170876","cluster_addr":"172.21.15.189:6827/764170876","heartbeat_back_addr":"172.21.15.189:6831/764170876","heartbeat_front_addr":"172.21.15.189:6829/764170876","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-07T14:33:44.598937+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-07T14:34:07.818529+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-07T14:34:32.826953+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-07T14:34:56.635361+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-07T14:35:18.427639+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-07T14:35:40.181583+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-07T14:36:02.709930+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-07T14:36:26.467881+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.83:0/848305012":"2024-04-08T14:32:22.035463+0000","172.21.15.83:6800/3303842870":"2024-04-08T14:32:22.035463+0000","172.21.15.83:0/3248703110":"2024-04-08T14:32:22.035463+0000","172.21.15.83:0/1472449846":"2024-04-08T14:31:52.613300+0000","172.21.15.83:6800/533197939":"2024-04-08T14:31:52.613300+0000","172.21.15.83:6801/3303842870":"2024-04-08T14:32:22.035463+0000","172.21.15.83:6801/533197939":"2024-04-08T14:31:52.613300+0000","172.21.15.83:0/145418573":"2024-04-08T14:32:22.035463+0000","172.21.15.83:0/1879750037":"2024-04-08T14:31:52.613300+0000","172.21.15.83:0/1319892688":"2024-04-08T14:31:52.613300+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-07T14:36:34.877 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:34 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/1762960624' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-07T14:36:34.877 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:34 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/1762960624' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-07T14:36:34.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:34 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/1762960624' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-07T14:36:35.277 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-07T14:34:36.411287+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-04-07T14:36:35.278 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph osd pool get .mgr pg_num 2024-04-07T14:36:35.579 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:35 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:36:35] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:36:35.733 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-07T14:36:36.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:36 smithi083 ceph-mon[24810]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-04-07T14:36:36.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:36 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:36.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:36 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:36.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:36 smithi083 ceph-mon[29774]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-04-07T14:36:36.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:36 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:36.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:36 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:36.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:36 smithi189 ceph-mon[25797]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s, 0 objects/s recovering 2024-04-07T14:36:36.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:36 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:36.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:36 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:37.593 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:37 smithi083 ceph-mon[24810]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-04-07T14:36:37.593 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:37 smithi083 ceph-mon[29774]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-04-07T14:36:37.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:37 smithi189 ceph-mon[25797]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-04-07T14:36:39.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:39 smithi083 ceph-mon[24810]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-07T14:36:39.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:39 smithi083 ceph-mon[29774]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-07T14:36:39.555 INFO:teuthology.orchestra.run.smithi083.stdout:pg_num: 1 2024-04-07T14:36:39.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:39 smithi189 ceph-mon[25797]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-07T14:36:40.224 INFO:tasks.cephadm:Adding prometheus.a on smithi189 2024-04-07T14:36:40.225 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph orch apply prometheus '1;smithi189=a' 2024-04-07T14:36:40.549 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:40 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/770045446' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-07T14:36:40.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:40 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/770045446' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-07T14:36:40.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:40 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/770045446' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-07T14:36:41.553 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:41 smithi189 ceph-mon[25797]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-07T14:36:41.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:41 smithi083 ceph-mon[29774]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-07T14:36:41.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:41 smithi083 ceph-mon[24810]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-07T14:36:42.000 INFO:teuthology.orchestra.run.smithi189.stdout:Scheduled prometheus update... 2024-04-07T14:36:42.534 DEBUG:teuthology.orchestra.run.smithi189:prometheus.a> sudo journalctl -f -n 0 -u ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@prometheus.a.service 2024-04-07T14:36:42.537 INFO:tasks.cephadm:Adding node-exporter.a on smithi083 2024-04-07T14:36:42.537 INFO:tasks.cephadm:Adding node-exporter.b on smithi189 2024-04-07T14:36:42.537 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph orch apply node-exporter '2;smithi083=a;smithi189=b' 2024-04-07T14:36:43.131 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:36:43 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: ignoring --setuser ceph since I am not root 2024-04-07T14:36:43.131 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:36:43 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: ignoring --setgroup ceph since I am not root 2024-04-07T14:36:43.131 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:36:43 smithi189 ceph-mgr[27353]: -- 172.21.15.189:0/2415472715 <== mon.2 v2:172.21.15.189:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55f655fbe340 con 0x55f655f97000 2024-04-07T14:36:43.131 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:43 smithi189 ceph-mon[25797]: from='client.24320 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi189=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:36:43.132 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:43 smithi189 ceph-mon[25797]: Saving service prometheus spec with placement smithi189=a;count:1 2024-04-07T14:36:43.132 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:43 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:43.132 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:43 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:36:43.132 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:43 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:36:43.132 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:43 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:36:43.132 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:43 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:43.132 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:43 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-07T14:36:43.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:42 smithi083 ceph-mon[24810]: from='client.24320 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi189=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:36:43.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:42 smithi083 ceph-mon[24810]: Saving service prometheus spec with placement smithi189=a;count:1 2024-04-07T14:36:43.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:43 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:43.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:43 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:36:43.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:43 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:36:43.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:43 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:36:43.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:43 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:43.428 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:43 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-07T14:36:43.428 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:43 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ignoring --setuser ceph since I am not root 2024-04-07T14:36:43.428 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:43 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ignoring --setgroup ceph since I am not root 2024-04-07T14:36:43.429 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:43 smithi083 ceph-mgr[25049]: -- 172.21.15.83:0/1046511560 <== mon.1 v2:172.21.15.83:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5621aaeac340 con 0x5621aae84c00 2024-04-07T14:36:43.429 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:43 smithi083 ceph-mon[29774]: from='client.24320 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi189=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:36:43.429 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:43 smithi083 ceph-mon[29774]: Saving service prometheus spec with placement smithi189=a;count:1 2024-04-07T14:36:43.429 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:43 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:43.429 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:43 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:36:43.429 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:43 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:36:43.429 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:43 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:36:43.429 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:43 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' 2024-04-07T14:36:43.430 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:43 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-07T14:36:43.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:43 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:36:43.551+0000 7f0110c56000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-07T14:36:43.926 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:43 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:36:43.638+0000 7f0110c56000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-07T14:36:43.954 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:36:43 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:36:43.556+0000 7f9a53b5f000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-07T14:36:43.954 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:36:43 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:36:43.646+0000 7f9a53b5f000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-07T14:36:44.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:44 smithi189 ceph-mon[25797]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-07T14:36:44.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:44 smithi189 ceph-mon[25797]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-07T14:36:44.385 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:44 smithi083 ceph-mon[24810]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-07T14:36:44.385 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:44 smithi083 ceph-mon[24810]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-07T14:36:44.385 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:44 smithi083 ceph-mon[29774]: from='mgr.14150 172.21.15.83:0/899754817' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-07T14:36:44.385 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:44 smithi083 ceph-mon[29774]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-07T14:36:44.386 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:44 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:36:44.218+0000 7f0110c56000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-07T14:36:44.386 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:44 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:36:44.306+0000 7f0110c56000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-07T14:36:44.386 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:44 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:36:44.386+0000 7f0110c56000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-07T14:36:44.605 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:36:44 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:36:44.274+0000 7f9a53b5f000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-07T14:36:44.605 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:36:44 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:36:44.365+0000 7f9a53b5f000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-07T14:36:44.605 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:36:44 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:36:44.450+0000 7f9a53b5f000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-07T14:36:44.605 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:36:44 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:36:44.605+0000 7f9a53b5f000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-07T14:36:44.675 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:44 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:36:44.533+0000 7f0110c56000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-07T14:36:44.676 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:44 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:36:44.612+0000 7f0110c56000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-07T14:36:44.876 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:36:44 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:36:44.685+0000 7f9a53b5f000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-07T14:36:44.876 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:36:44 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:36:44.793+0000 7f9a53b5f000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-07T14:36:44.876 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:36:44 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:36:44.876+0000 7f9a53b5f000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-07T14:36:45.042 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:44 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:36:44.719+0000 7f0110c56000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-07T14:36:45.043 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:44 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:36:44.801+0000 7f0110c56000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-07T14:36:45.043 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:44 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:36:44.960+0000 7f0110c56000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-07T14:36:45.043 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:45 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:36:45.044+0000 7f0110c56000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-07T14:36:45.128 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:36:45 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:36:45.041+0000 7f9a53b5f000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-07T14:36:45.129 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:36:45 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:36:45.128+0000 7f9a53b5f000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-07T14:36:46.372 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:46 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:36:46.059+0000 7f0110c56000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-07T14:36:46.372 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:46 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:36:46.188+0000 7f0110c56000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-07T14:36:46.458 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:36:46 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:36:46.148+0000 7f9a53b5f000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-07T14:36:46.458 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:36:46 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:36:46.278+0000 7f9a53b5f000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-07T14:36:46.675 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:46 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:36:46.373+0000 7f0110c56000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-07T14:36:46.676 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:46 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:36:46.459+0000 7f0110c56000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-07T14:36:46.676 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:46 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:36:46.592+0000 7f0110c56000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-07T14:36:46.721 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:36:46 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:36:46.468+0000 7f9a53b5f000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-07T14:36:46.722 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:36:46 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:36:46.575+0000 7f9a53b5f000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-07T14:36:46.722 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:36:46 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:36:46.721+0000 7f9a53b5f000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-07T14:36:47.175 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:46 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:36:46.908+0000 7f0110c56000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-07T14:36:47.176 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:47 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:36:47.125+0000 7f0110c56000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-07T14:36:47.347 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:36:47 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:36:47.041+0000 7f9a53b5f000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-07T14:36:47.347 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:36:47 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:36:47.261+0000 7f9a53b5f000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-07T14:36:47.347 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:36:47 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:36:47.347+0000 7f9a53b5f000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-07T14:36:47.675 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:47 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:36:47.209+0000 7f0110c56000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-07T14:36:47.675 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:47 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:36:47.377+0000 7f0110c56000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-07T14:36:47.708 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:36:47 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:36:47.515+0000 7f9a53b5f000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-07T14:36:48.570 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:48 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:36:48.281+0000 7f0110c56000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-07T14:36:48.619 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:48 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:36:48.572+0000 7f0110c56000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-07T14:36:48.708 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:36:48 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:36:48.440+0000 7f9a53b5f000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-07T14:36:48.886 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:48 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:36:48.666+0000 7f0110c56000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-07T14:36:48.886 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:48 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:36:48.815+0000 7f0110c56000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-07T14:36:48.980 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:48 smithi189 ceph-mon[25797]: Active manager daemon y restarted 2024-04-07T14:36:48.980 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:48 smithi189 ceph-mon[25797]: Activating manager daemon y 2024-04-07T14:36:48.981 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:48 smithi189 ceph-mon[25797]: osdmap e60: 8 total, 8 up, 8 in 2024-04-07T14:36:48.981 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:36:48 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:36:48.740+0000 7f9a53b5f000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-07T14:36:48.981 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:36:48 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:36:48.839+0000 7f9a53b5f000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-07T14:36:49.175 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:48 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: [07/Apr/2024:14:36:48] ENGINE Bus STARTING 2024-04-07T14:36:49.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:48 smithi083 ceph-mon[24810]: Active manager daemon y restarted 2024-04-07T14:36:49.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:48 smithi083 ceph-mon[24810]: Activating manager daemon y 2024-04-07T14:36:49.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:48 smithi083 ceph-mon[24810]: osdmap e60: 8 total, 8 up, 8 in 2024-04-07T14:36:49.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:48 smithi083 ceph-mon[29774]: Active manager daemon y restarted 2024-04-07T14:36:49.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:48 smithi083 ceph-mon[29774]: Activating manager daemon y 2024-04-07T14:36:49.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:48 smithi083 ceph-mon[29774]: osdmap e60: 8 total, 8 up, 8 in 2024-04-07T14:36:49.261 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:36:49.002+0000 7f9a53b5f000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-07T14:36:49.261 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: [07/Apr/2024:14:36:49] ENGINE Bus STARTING 2024-04-07T14:36:49.261 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: CherryPy Checker: 2024-04-07T14:36:49.261 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: The Application mounted at '' has an empty config. 2024-04-07T14:36:49.261 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: 2024-04-07T14:36:49.261 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: [07/Apr/2024:14:36:49] ENGINE Serving on http://:::9283 2024-04-07T14:36:49.261 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: [07/Apr/2024:14:36:49] ENGINE Bus STARTED 2024-04-07T14:36:49.500 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: CherryPy Checker: 2024-04-07T14:36:49.500 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: The Application mounted at '' has an empty config. 2024-04-07T14:36:49.500 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:36:49.500 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: [07/Apr/2024:14:36:49] ENGINE Serving on http://:::9283 2024-04-07T14:36:49.501 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: [07/Apr/2024:14:36:49] ENGINE Bus STARTED 2024-04-07T14:36:49.911 INFO:teuthology.orchestra.run.smithi189.stdout:Scheduled node-exporter update... 2024-04-07T14:36:50.175 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: [07/Apr/2024:14:36:49] ENGINE Bus STARTING 2024-04-07T14:36:50.176 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:50 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: [07/Apr/2024:14:36:50] ENGINE Serving on https://172.21.15.83:7150 2024-04-07T14:36:50.176 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:50 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: [07/Apr/2024:14:36:50] ENGINE Bus STARTED 2024-04-07T14:36:50.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[24810]: mgrmap e17: y(active, starting, since 0.0558904s), standbys: x 2024-04-07T14:36:50.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-07T14:36:50.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-07T14:36:50.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-07T14:36:50.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-07T14:36:50.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-07T14:36:50.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-07T14:36:50.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-07T14:36:50.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-07T14:36:50.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-07T14:36:50.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-07T14:36:50.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-07T14:36:50.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-07T14:36:50.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-07T14:36:50.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-07T14:36:50.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-07T14:36:50.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-07T14:36:50.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[24810]: Manager daemon y is now available 2024-04-07T14:36:50.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:36:50.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:36:50.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:36:50.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:36:50.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:36:50.178 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi189", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:36:50.178 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi083", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:36:50.178 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd='[{"prefix": "config rm", "who": "osd/host:smithi189", "name": "osd_memory_target"}]': finished 2024-04-07T14:36:50.178 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[24810]: Updating smithi189:/etc/ceph/ceph.conf 2024-04-07T14:36:50.178 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[29774]: mgrmap e17: y(active, starting, since 0.0558904s), standbys: x 2024-04-07T14:36:50.178 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-07T14:36:50.178 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-07T14:36:50.178 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-07T14:36:50.178 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-07T14:36:50.178 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-07T14:36:50.178 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-07T14:36:50.179 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-07T14:36:50.179 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-07T14:36:50.179 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-07T14:36:50.179 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-07T14:36:50.179 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-07T14:36:50.179 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-07T14:36:50.179 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-07T14:36:50.179 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-07T14:36:50.179 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-07T14:36:50.179 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-07T14:36:50.179 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[29774]: Manager daemon y is now available 2024-04-07T14:36:50.179 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:36:50.179 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:36:50.179 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:36:50.179 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:36:50.179 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:36:50.180 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi189", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:36:50.180 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi083", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:36:50.180 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd='[{"prefix": "config rm", "who": "osd/host:smithi189", "name": "osd_memory_target"}]': finished 2024-04-07T14:36:50.180 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[29774]: Updating smithi189:/etc/ceph/ceph.conf 2024-04-07T14:36:50.180 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd='[{"prefix": "config rm", "who": "osd/host:smithi083", "name": "osd_memory_target"}]': finished 2024-04-07T14:36:50.180 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[24810]: Updating smithi083:/etc/ceph/ceph.conf 2024-04-07T14:36:50.180 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:36:50.180 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:36:50.180 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:36:50.180 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[24810]: Standby manager daemon x restarted 2024-04-07T14:36:50.180 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[24810]: Standby manager daemon x started 2024-04-07T14:36:50.180 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[24810]: from='mgr.? 172.21.15.189:0/3874212346' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-07T14:36:50.181 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[24810]: from='mgr.? 172.21.15.189:0/3874212346' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-07T14:36:50.181 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[24810]: from='mgr.? 172.21.15.189:0/3874212346' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-07T14:36:50.181 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[24810]: from='mgr.? 172.21.15.189:0/3874212346' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-07T14:36:50.181 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd='[{"prefix": "config rm", "who": "osd/host:smithi083", "name": "osd_memory_target"}]': finished 2024-04-07T14:36:50.181 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[29774]: Updating smithi083:/etc/ceph/ceph.conf 2024-04-07T14:36:50.181 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:36:50.181 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:36:50.181 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:36:50.181 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[29774]: Standby manager daemon x restarted 2024-04-07T14:36:50.181 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[29774]: Standby manager daemon x started 2024-04-07T14:36:50.181 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[29774]: from='mgr.? 172.21.15.189:0/3874212346' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-07T14:36:50.181 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[29774]: from='mgr.? 172.21.15.189:0/3874212346' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-07T14:36:50.181 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[29774]: from='mgr.? 172.21.15.189:0/3874212346' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-07T14:36:50.181 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:49 smithi083 ceph-mon[29774]: from='mgr.? 172.21.15.189:0/3874212346' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-07T14:36:50.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-mon[25797]: mgrmap e17: y(active, starting, since 0.0558904s), standbys: x 2024-04-07T14:36:50.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-07T14:36:50.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-07T14:36:50.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-07T14:36:50.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-07T14:36:50.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-07T14:36:50.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-07T14:36:50.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-07T14:36:50.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-07T14:36:50.210 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-07T14:36:50.210 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-07T14:36:50.210 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-07T14:36:50.210 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-07T14:36:50.210 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-07T14:36:50.210 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-07T14:36:50.210 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-07T14:36:50.210 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-07T14:36:50.210 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-mon[25797]: Manager daemon y is now available 2024-04-07T14:36:50.210 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:36:50.210 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:36:50.210 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:36:50.210 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:36:50.210 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:36:50.210 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi189", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:36:50.211 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi083", "name": "osd_memory_target"}]: dispatch 2024-04-07T14:36:50.211 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd='[{"prefix": "config rm", "who": "osd/host:smithi189", "name": "osd_memory_target"}]': finished 2024-04-07T14:36:50.211 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-mon[25797]: Updating smithi189:/etc/ceph/ceph.conf 2024-04-07T14:36:50.211 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd='[{"prefix": "config rm", "who": "osd/host:smithi083", "name": "osd_memory_target"}]': finished 2024-04-07T14:36:50.211 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-mon[25797]: Updating smithi083:/etc/ceph/ceph.conf 2024-04-07T14:36:50.211 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:36:50.211 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:36:50.211 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:36:50.211 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-mon[25797]: Standby manager daemon x restarted 2024-04-07T14:36:50.211 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-mon[25797]: Standby manager daemon x started 2024-04-07T14:36:50.211 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-mon[25797]: from='mgr.? 172.21.15.189:0/3874212346' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-07T14:36:50.211 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-mon[25797]: from='mgr.? 172.21.15.189:0/3874212346' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-07T14:36:50.211 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-mon[25797]: from='mgr.? 172.21.15.189:0/3874212346' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-07T14:36:50.211 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:49 smithi189 ceph-mon[25797]: from='mgr.? 172.21.15.189:0/3874212346' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-07T14:36:50.710 DEBUG:teuthology.orchestra.run.smithi083:node-exporter.a> sudo journalctl -f -n 0 -u ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@node-exporter.a.service 2024-04-07T14:36:50.713 DEBUG:teuthology.orchestra.run.smithi189:node-exporter.b> sudo journalctl -f -n 0 -u ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@node-exporter.b.service 2024-04-07T14:36:50.715 INFO:tasks.cephadm:Adding alertmanager.a on smithi083 2024-04-07T14:36:50.716 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph orch apply alertmanager '1;smithi083=a' 2024-04-07T14:36:51.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:50 smithi083 ceph-mon[24810]: Updating smithi083:/etc/ceph/ceph.client.admin.keyring 2024-04-07T14:36:51.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:50 smithi083 ceph-mon[24810]: Updating smithi189:/etc/ceph/ceph.client.admin.keyring 2024-04-07T14:36:51.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:50 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:36:51.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:50 smithi083 ceph-mon[24810]: [07/Apr/2024:14:36:49] ENGINE Bus STARTING 2024-04-07T14:36:51.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:50 smithi083 ceph-mon[24810]: mgrmap e18: y(active, since 1.06697s), standbys: x 2024-04-07T14:36:51.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:50 smithi083 ceph-mon[24810]: from='client.24338 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi083=a;smithi189=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:36:51.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:50 smithi083 ceph-mon[24810]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:36:51.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:50 smithi083 ceph-mon[24810]: Saving service node-exporter spec with placement smithi083=a;smithi189=b;count:2 2024-04-07T14:36:51.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:50 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:36:51.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:50 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:36:51.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:50 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:36:51.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:50 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-07T14:36:51.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:50 smithi083 ceph-mon[24810]: [07/Apr/2024:14:36:50] ENGINE Serving on https://172.21.15.83:7150 2024-04-07T14:36:51.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:50 smithi083 ceph-mon[24810]: [07/Apr/2024:14:36:50] ENGINE Bus STARTED 2024-04-07T14:36:51.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:50 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:36:51.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:50 smithi083 ceph-mon[29774]: Updating smithi083:/etc/ceph/ceph.client.admin.keyring 2024-04-07T14:36:51.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:50 smithi083 ceph-mon[29774]: Updating smithi189:/etc/ceph/ceph.client.admin.keyring 2024-04-07T14:36:51.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:50 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:36:51.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:50 smithi083 ceph-mon[29774]: [07/Apr/2024:14:36:49] ENGINE Bus STARTING 2024-04-07T14:36:51.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:50 smithi083 ceph-mon[29774]: mgrmap e18: y(active, since 1.06697s), standbys: x 2024-04-07T14:36:51.178 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:50 smithi083 ceph-mon[29774]: from='client.24338 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi083=a;smithi189=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:36:51.178 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:50 smithi083 ceph-mon[29774]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:36:51.178 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:50 smithi083 ceph-mon[29774]: Saving service node-exporter spec with placement smithi083=a;smithi189=b;count:2 2024-04-07T14:36:51.178 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:50 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:36:51.178 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:50 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:36:51.178 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:50 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:36:51.178 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:50 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-07T14:36:51.178 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:50 smithi083 ceph-mon[29774]: [07/Apr/2024:14:36:50] ENGINE Serving on https://172.21.15.83:7150 2024-04-07T14:36:51.178 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:50 smithi083 ceph-mon[29774]: [07/Apr/2024:14:36:50] ENGINE Bus STARTED 2024-04-07T14:36:51.178 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:50 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:36:51.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:50 smithi189 ceph-mon[25797]: Updating smithi083:/etc/ceph/ceph.client.admin.keyring 2024-04-07T14:36:51.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:50 smithi189 ceph-mon[25797]: Updating smithi189:/etc/ceph/ceph.client.admin.keyring 2024-04-07T14:36:51.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:50 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:36:51.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:50 smithi189 ceph-mon[25797]: [07/Apr/2024:14:36:49] ENGINE Bus STARTING 2024-04-07T14:36:51.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:50 smithi189 ceph-mon[25797]: mgrmap e18: y(active, since 1.06697s), standbys: x 2024-04-07T14:36:51.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:50 smithi189 ceph-mon[25797]: from='client.24338 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi083=a;smithi189=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:36:51.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:50 smithi189 ceph-mon[25797]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:36:51.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:50 smithi189 ceph-mon[25797]: Saving service node-exporter spec with placement smithi083=a;smithi189=b;count:2 2024-04-07T14:36:51.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:50 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:36:51.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:50 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:36:51.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:50 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:36:51.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:50 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-07T14:36:51.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:50 smithi189 ceph-mon[25797]: [07/Apr/2024:14:36:50] ENGINE Serving on https://172.21.15.83:7150 2024-04-07T14:36:51.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:50 smithi189 ceph-mon[25797]: [07/Apr/2024:14:36:50] ENGINE Bus STARTED 2024-04-07T14:36:51.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:50 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:36:52.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:51 smithi083 ceph-mon[24810]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:36:52.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:51 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-07T14:36:52.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:51 smithi083 ceph-mon[24810]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-07T14:36:52.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:51 smithi083 ceph-mon[24810]: Deploying daemon prometheus.a on smithi189 2024-04-07T14:36:52.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:51 smithi083 ceph-mon[29774]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:36:52.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:51 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-07T14:36:52.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:51 smithi083 ceph-mon[29774]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-07T14:36:52.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:51 smithi083 ceph-mon[29774]: Deploying daemon prometheus.a on smithi189 2024-04-07T14:36:52.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:51 smithi189 ceph-mon[25797]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:36:52.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:51 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-07T14:36:52.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:51 smithi189 ceph-mon[25797]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-07T14:36:52.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:51 smithi189 ceph-mon[25797]: Deploying daemon prometheus.a on smithi189 2024-04-07T14:36:53.169 INFO:teuthology.orchestra.run.smithi189.stdout:Scheduled alertmanager update... 2024-04-07T14:36:54.111 DEBUG:teuthology.orchestra.run.smithi083:alertmanager.a> sudo journalctl -f -n 0 -u ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@alertmanager.a.service 2024-04-07T14:36:54.114 INFO:tasks.cephadm:Adding grafana.a on smithi189 2024-04-07T14:36:54.114 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph orch apply grafana '1;smithi189=a' 2024-04-07T14:36:54.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:53 smithi083 ceph-mon[29774]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:36:54.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:53 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:36:54.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:53 smithi083 ceph-mon[24810]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:36:54.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:53 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:36:54.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:53 smithi189 ceph-mon[25797]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:36:54.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:53 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:36:55.168 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:54 smithi189 ceph-mon[25797]: from='client.24368 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi083=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:36:55.168 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:54 smithi189 ceph-mon[25797]: Saving service alertmanager spec with placement smithi083=a;count:1 2024-04-07T14:36:55.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:54 smithi083 ceph-mon[24810]: from='client.24368 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi083=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:36:55.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:54 smithi083 ceph-mon[24810]: Saving service alertmanager spec with placement smithi083=a;count:1 2024-04-07T14:36:55.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:54 smithi083 ceph-mon[29774]: from='client.24368 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi083=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:36:55.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:54 smithi083 ceph-mon[29774]: Saving service alertmanager spec with placement smithi083=a;count:1 2024-04-07T14:36:55.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:36:55 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:36:55] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:36:56.100 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:55 smithi189 ceph-mon[25797]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:36:56.100 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:55 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:36:56.100 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:55 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:36:56.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:55 smithi083 ceph-mon[24810]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:36:56.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:55 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:36:56.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:55 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:36:56.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:55 smithi083 ceph-mon[29774]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:36:56.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:55 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:36:56.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:55 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:36:57.447 INFO:teuthology.orchestra.run.smithi189.stdout:Scheduled grafana update... 2024-04-07T14:36:58.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:57 smithi083 ceph-mon[24810]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:36:58.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:57 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:36:58.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:57 smithi083 ceph-mon[29774]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:36:58.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:57 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:36:58.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:57 smithi189 ceph-mon[25797]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:36:58.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:57 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:36:58.709 DEBUG:teuthology.orchestra.run.smithi189:grafana.a> sudo journalctl -f -n 0 -u ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@grafana.a.service 2024-04-07T14:36:58.712 INFO:tasks.cephadm:Setting up client nodes... 2024-04-07T14:36:58.712 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-07T14:36:59.149 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:58 smithi083 ceph-mon[24810]: from='client.24374 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi189=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:36:59.150 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:58 smithi083 ceph-mon[24810]: Saving service grafana spec with placement smithi189=a;count:1 2024-04-07T14:36:59.150 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:58 smithi083 ceph-mon[29774]: from='client.24374 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi189=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:36:59.150 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:58 smithi083 ceph-mon[29774]: Saving service grafana spec with placement smithi189=a;count:1 2024-04-07T14:36:59.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:58 smithi189 ceph-mon[25797]: from='client.24374 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi189=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:36:59.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:58 smithi189 ceph-mon[25797]: Saving service grafana spec with placement smithi189=a;count:1 2024-04-07T14:37:00.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:36:59 smithi083 ceph-mon[24810]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:00.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:36:59 smithi083 ceph-mon[29774]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:00.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:36:59 smithi189 ceph-mon[25797]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:01.350 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:01 smithi189 ceph-mon[25797]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:01.409 INFO:teuthology.orchestra.run.smithi083.stdout:[client.0] 2024-04-07T14:37:01.409 INFO:teuthology.orchestra.run.smithi083.stdout: key = AQCNrxJmty08GBAAfjcJx1/NML8sLouw1gCOOg== 2024-04-07T14:37:01.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:01 smithi083 ceph-mon[24810]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:01.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:01 smithi083 ceph-mon[29774]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:02.131 DEBUG:teuthology.orchestra.run.smithi083:> set -ex 2024-04-07T14:37:02.131 DEBUG:teuthology.orchestra.run.smithi083:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-04-07T14:37:02.131 DEBUG:teuthology.orchestra.run.smithi083:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-04-07T14:37:02.176 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-07T14:37:02.324 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:02 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/1422267446' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-07T14:37:02.324 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:02 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/1422267446' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-07T14:37:02.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:02 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/1422267446' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-07T14:37:02.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:02 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/1422267446' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-07T14:37:02.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:02 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/1422267446' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-07T14:37:02.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:02 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/1422267446' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-07T14:37:02.619 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:02 smithi189 systemd[1]: Starting Ceph prometheus.a for 7388281a-f4eb-11ee-b649-cb9ed24678a4... 2024-04-07T14:37:03.494 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:03 smithi189 ceph-mon[25797]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:03.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:03 smithi083 ceph-mon[24810]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:03.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:03 smithi083 ceph-mon[29774]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:04.250 INFO:teuthology.orchestra.run.smithi189.stdout:[client.1] 2024-04-07T14:37:04.250 INFO:teuthology.orchestra.run.smithi189.stdout: key = AQCQrxJmFra4DhAAkj287V45jSWC9tysdjv1wQ== 2024-04-07T14:37:04.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:04 smithi189 ceph-mon[25797]: from='client.? 172.21.15.189:0/2191375262' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-07T14:37:04.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:04 smithi189 ceph-mon[25797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-07T14:37:04.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:04 smithi189 ceph-mon[25797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-07T14:37:04.459 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:04 smithi189 podman[42005]: 2024-04-07 14:37:04.091218982 +0000 UTC m=+0.018204480 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-07T14:37:04.459 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:04 smithi189 podman[42005]: 2024-04-07 14:37:04.192063667 +0000 UTC m=+0.119049152 container create ac215bd79e01b73d669acb41c2a26d86c09ec88e71db0e579c34732d9048052a (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-07T14:37:04.577 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:04 smithi083 ceph-mon[24810]: from='client.? 172.21.15.189:0/2191375262' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-07T14:37:04.577 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:04 smithi083 ceph-mon[24810]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-07T14:37:04.577 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:04 smithi083 ceph-mon[24810]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-07T14:37:04.577 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:04 smithi083 ceph-mon[29774]: from='client.? 172.21.15.189:0/2191375262' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-07T14:37:04.577 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:04 smithi083 ceph-mon[29774]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-07T14:37:04.577 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:04 smithi083 ceph-mon[29774]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-07T14:37:04.835 DEBUG:teuthology.orchestra.run.smithi189:> set -ex 2024-04-07T14:37:04.836 DEBUG:teuthology.orchestra.run.smithi189:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-04-07T14:37:04.836 DEBUG:teuthology.orchestra.run.smithi189:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-04-07T14:37:04.881 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-07T14:37:04.881 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-07T14:37:04.882 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph mgr dump --format=json 2024-04-07T14:37:05.089 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:04 smithi189 podman[42005]: 2024-04-07 14:37:04.964594028 +0000 UTC m=+0.891579514 container init ac215bd79e01b73d669acb41c2a26d86c09ec88e71db0e579c34732d9048052a (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-07T14:37:05.089 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:04 smithi189 podman[42005]: 2024-04-07 14:37:04.968261988 +0000 UTC m=+0.895247474 container start ac215bd79e01b73d669acb41c2a26d86c09ec88e71db0e579c34732d9048052a (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-07T14:37:05.089 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:04 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[42032]: ts=2024-04-07T14:37:04.991Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-07T14:37:05.089 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:04 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[42032]: ts=2024-04-07T14:37:04.991Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-07T14:37:05.089 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:04 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[42032]: ts=2024-04-07T14:37:04.991Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-07T14:37:05.089 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:04 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[42032]: ts=2024-04-07T14:37:04.991Z caller=main.go:518 level=info host_details="(Linux 5.14.0-432.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Wed Mar 20 20:26:59 UTC 2024 x86_64 smithi189 (none))" 2024-04-07T14:37:05.089 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:04 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[42032]: ts=2024-04-07T14:37:04.991Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-07T14:37:05.089 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:04 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[42032]: ts=2024-04-07T14:37:04.991Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-07T14:37:05.089 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:04 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[42032]: ts=2024-04-07T14:37:04.993Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-07T14:37:05.089 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:04 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[42032]: ts=2024-04-07T14:37:04.993Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-07T14:37:05.089 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:04 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[42032]: ts=2024-04-07T14:37:04.994Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-07T14:37:05.090 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:04 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[42032]: ts=2024-04-07T14:37:04.997Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-07T14:37:05.090 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:04 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[42032]: ts=2024-04-07T14:37:04.997Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.63µs 2024-04-07T14:37:05.090 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:04 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[42032]: ts=2024-04-07T14:37:04.997Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-07T14:37:05.090 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:04 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[42032]: ts=2024-04-07T14:37:04.997Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-04-07T14:37:05.090 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:04 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[42032]: ts=2024-04-07T14:37:04.997Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=20.769µs wal_replay_duration=152.951µs total_replay_duration=187.688µs 2024-04-07T14:37:05.090 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:04 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[42032]: ts=2024-04-07T14:37:04.998Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-07T14:37:05.090 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:04 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[42032]: ts=2024-04-07T14:37:04.998Z caller=main.go:947 level=info msg="TSDB started" 2024-04-07T14:37:05.090 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:04 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[42032]: ts=2024-04-07T14:37:04.998Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-07T14:37:05.090 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:05 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[42032]: ts=2024-04-07T14:37:05.014Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.765953ms db_storage=654ns remote_storage=1.152µs web_handler=231ns query_engine=479ns scrape=377.036µs scrape_sd=15.759µs notify=731ns notify_sd=1.378µs rules=15.196082ms 2024-04-07T14:37:05.090 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:05 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[42032]: ts=2024-04-07T14:37:05.014Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-07T14:37:05.090 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:05 smithi189 bash[42005]: ac215bd79e01b73d669acb41c2a26d86c09ec88e71db0e579c34732d9048052a 2024-04-07T14:37:05.090 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:05 smithi189 systemd[1]: Started Ceph prometheus.a for 7388281a-f4eb-11ee-b649-cb9ed24678a4. 2024-04-07T14:37:05.106 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-07T14:37:06.381 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:06 smithi083 ceph-mon[24810]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:06.382 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:06 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:06.382 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:06 smithi083 ceph-mon[24810]: Deploying daemon node-exporter.a on smithi083 2024-04-07T14:37:06.382 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:06 smithi083 ceph-mon[29774]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:06.382 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:06 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:06.382 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:06 smithi083 ceph-mon[29774]: Deploying daemon node-exporter.a on smithi083 2024-04-07T14:37:06.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:06 smithi189 ceph-mon[25797]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:06.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:06 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:06.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:06 smithi189 ceph-mon[25797]: Deploying daemon node-exporter.a on smithi083 2024-04-07T14:37:06.926 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:37:06 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:37:06] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:37:07.151 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-07T14:37:08.088 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":19,"active_gid":24358,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6800","nonce":3162313196},{"type":"v1","addr":"172.21.15.83:6801","nonce":3162313196}]},"active_addr":"172.21.15.83:6801/3162313196","active_change":"2024-04-07T14:36:48.830266+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":24326,"name":"x","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.83:8443/","prometheus":"http://172.21.15.83:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":60,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.83:0","nonce":3574418593}]},{"addrvec":[{"type":"v2","addr":"172.21.15.83:0","nonce":2674292070}]},{"addrvec":[{"type":"v2","addr":"172.21.15.83:0","nonce":2871420407}]},{"addrvec":[{"type":"v2","addr":"172.21.15.83:0","nonce":3906962109}]}]}} 2024-04-07T14:37:08.092 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-07T14:37:08.092 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-07T14:37:08.092 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-07T14:37:08.099 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:07 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:08.099 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:07 smithi189 ceph-mon[25797]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:08.101 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:07 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/792702321' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-07T14:37:08.117 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:07 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:08.117 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:07 smithi083 ceph-mon[24810]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:08.118 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:07 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/792702321' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-07T14:37:08.118 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:07 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:08.118 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:07 smithi083 ceph-mon[29774]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:08.118 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:07 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/792702321' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-07T14:37:08.307 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-07T14:37:09.676 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:37:09 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:37:09] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:37:10.160 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:09 smithi083 ceph-mon[24810]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:10.160 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:09 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:10.160 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:09 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:10.160 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:09 smithi083 ceph-mon[29774]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:10.161 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:09 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:10.161 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:09 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:10.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:09 smithi189 ceph-mon[25797]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:10.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:09 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:10.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:09 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:10.425 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:37:10 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:37:10] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:37:10.425 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:37:10 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:37:10] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:37:10.798 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Apr 07 14:37:10 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-a[45835]: ts=2024-04-07T14:37:10.707Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-04-07T14:37:10.798 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Apr 07 14:37:10 smithi083 bash[45669]: a8c27029d23c28d14db040d628deab5a8dcd1184760a15b0c9561adfbdbe511c 2024-04-07T14:37:11.066 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Apr 07 14:37:10 smithi083 systemd[1]: Started Ceph node-exporter.a for 7388281a-f4eb-11ee-b649-cb9ed24678a4. 2024-04-07T14:37:11.067 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:10 smithi083 ceph-mon[24810]: Detected new or changed devices on smithi189 2024-04-07T14:37:11.067 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:10 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:11.067 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:10 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:11.067 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:10 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:11.067 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:10 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:11.067 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:10 smithi083 ceph-mon[29774]: Detected new or changed devices on smithi189 2024-04-07T14:37:11.067 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:10 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:11.067 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:10 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:11.068 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:10 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:11.068 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:10 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:11.175 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:10 smithi189 ceph-mon[25797]: Detected new or changed devices on smithi189 2024-04-07T14:37:11.175 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:10 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:11.175 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:10 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:11.175 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:10 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:11.175 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:10 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:11.458 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-07T14:37:11.458 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":60,"fsid":"7388281a-f4eb-11ee-b649-cb9ed24678a4","created":"2024-04-07T14:31:30.809811+0000","modified":"2024-04-07T14:36:48.829956+0000","last_up_change":"2024-04-07T14:36:28.289077+0000","last_in_change":"2024-04-07T14:36:10.041386+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-07T14:34:36.411287+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"a42cc075-c4af-4500-abae-738cfd1c8d97","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":159265472},{"type":"v1","addr":"172.21.15.83:6803","nonce":159265472}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":159265472},{"type":"v1","addr":"172.21.15.83:6805","nonce":159265472}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":159265472},{"type":"v1","addr":"172.21.15.83:6809","nonce":159265472}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":159265472},{"type":"v1","addr":"172.21.15.83:6807","nonce":159265472}]},"public_addr":"172.21.15.83:6803/159265472","cluster_addr":"172.21.15.83:6805/159265472","heartbeat_back_addr":"172.21.15.83:6809/159265472","heartbeat_front_addr":"172.21.15.83:6807/159265472","state":["exists","up"]},{"osd":1,"uuid":"c0b76e30-ff29-43b7-a8d1-90f9bef15b7e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6810","nonce":4165704838},{"type":"v1","addr":"172.21.15.83:6811","nonce":4165704838}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6812","nonce":4165704838},{"type":"v1","addr":"172.21.15.83:6813","nonce":4165704838}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6816","nonce":4165704838},{"type":"v1","addr":"172.21.15.83:6817","nonce":4165704838}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6814","nonce":4165704838},{"type":"v1","addr":"172.21.15.83:6815","nonce":4165704838}]},"public_addr":"172.21.15.83:6811/4165704838","cluster_addr":"172.21.15.83:6813/4165704838","heartbeat_back_addr":"172.21.15.83:6817/4165704838","heartbeat_front_addr":"172.21.15.83:6815/4165704838","state":["exists","up"]},{"osd":2,"uuid":"fe1ee70e-866b-463e-b7fd-2a2057f44b80","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6818","nonce":718005878},{"type":"v1","addr":"172.21.15.83:6819","nonce":718005878}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6820","nonce":718005878},{"type":"v1","addr":"172.21.15.83:6821","nonce":718005878}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6824","nonce":718005878},{"type":"v1","addr":"172.21.15.83:6825","nonce":718005878}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6822","nonce":718005878},{"type":"v1","addr":"172.21.15.83:6823","nonce":718005878}]},"public_addr":"172.21.15.83:6819/718005878","cluster_addr":"172.21.15.83:6821/718005878","heartbeat_back_addr":"172.21.15.83:6825/718005878","heartbeat_front_addr":"172.21.15.83:6823/718005878","state":["exists","up"]},{"osd":3,"uuid":"14bfed5e-6a77-4448-855b-258e622311ee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6826","nonce":3079630945},{"type":"v1","addr":"172.21.15.83:6827","nonce":3079630945}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6828","nonce":3079630945},{"type":"v1","addr":"172.21.15.83:6829","nonce":3079630945}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6832","nonce":3079630945},{"type":"v1","addr":"172.21.15.83:6833","nonce":3079630945}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6830","nonce":3079630945},{"type":"v1","addr":"172.21.15.83:6831","nonce":3079630945}]},"public_addr":"172.21.15.83:6827/3079630945","cluster_addr":"172.21.15.83:6829/3079630945","heartbeat_back_addr":"172.21.15.83:6833/3079630945","heartbeat_front_addr":"172.21.15.83:6831/3079630945","state":["exists","up"]},{"osd":4,"uuid":"6a8687be-8c65-40a3-89c4-d463ed60d920","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6800","nonce":3355623828},{"type":"v1","addr":"172.21.15.189:6801","nonce":3355623828}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6802","nonce":3355623828},{"type":"v1","addr":"172.21.15.189:6803","nonce":3355623828}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6806","nonce":3355623828},{"type":"v1","addr":"172.21.15.189:6807","nonce":3355623828}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6804","nonce":3355623828},{"type":"v1","addr":"172.21.15.189:6805","nonce":3355623828}]},"public_addr":"172.21.15.189:6801/3355623828","cluster_addr":"172.21.15.189:6803/3355623828","heartbeat_back_addr":"172.21.15.189:6807/3355623828","heartbeat_front_addr":"172.21.15.189:6805/3355623828","state":["exists","up"]},{"osd":5,"uuid":"7115ab07-1339-4eac-b403-bf6b40361af9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6808","nonce":3207731440},{"type":"v1","addr":"172.21.15.189:6809","nonce":3207731440}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6810","nonce":3207731440},{"type":"v1","addr":"172.21.15.189:6811","nonce":3207731440}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6814","nonce":3207731440},{"type":"v1","addr":"172.21.15.189:6815","nonce":3207731440}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6812","nonce":3207731440},{"type":"v1","addr":"172.21.15.189:6813","nonce":3207731440}]},"public_addr":"172.21.15.189:6809/3207731440","cluster_addr":"172.21.15.189:6811/3207731440","heartbeat_back_addr":"172.21.15.189:6815/3207731440","heartbeat_front_addr":"172.21.15.189:6813/3207731440","state":["exists","up"]},{"osd":6,"uuid":"1411a622-2b3f-48f8-b265-8b58643679f0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6816","nonce":3839485321},{"type":"v1","addr":"172.21.15.189:6817","nonce":3839485321}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6818","nonce":3839485321},{"type":"v1","addr":"172.21.15.189:6819","nonce":3839485321}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6822","nonce":3839485321},{"type":"v1","addr":"172.21.15.189:6823","nonce":3839485321}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6820","nonce":3839485321},{"type":"v1","addr":"172.21.15.189:6821","nonce":3839485321}]},"public_addr":"172.21.15.189:6817/3839485321","cluster_addr":"172.21.15.189:6819/3839485321","heartbeat_back_addr":"172.21.15.189:6823/3839485321","heartbeat_front_addr":"172.21.15.189:6821/3839485321","state":["exists","up"]},{"osd":7,"uuid":"bbaaa4cf-7ff3-4a75-aeb8-d8325b9313c1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6824","nonce":764170876},{"type":"v1","addr":"172.21.15.189:6825","nonce":764170876}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6826","nonce":764170876},{"type":"v1","addr":"172.21.15.189:6827","nonce":764170876}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6830","nonce":764170876},{"type":"v1","addr":"172.21.15.189:6831","nonce":764170876}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6828","nonce":764170876},{"type":"v1","addr":"172.21.15.189:6829","nonce":764170876}]},"public_addr":"172.21.15.189:6825/764170876","cluster_addr":"172.21.15.189:6827/764170876","heartbeat_back_addr":"172.21.15.189:6831/764170876","heartbeat_front_addr":"172.21.15.189:6829/764170876","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-07T14:33:44.598937+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-07T14:34:07.818529+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-07T14:34:32.826953+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-07T14:34:56.635361+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-07T14:35:18.427639+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-07T14:35:40.181583+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-07T14:36:02.709930+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-07T14:36:26.467881+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.83:0/248923343":"2024-04-08T14:36:48.829922+0000","172.21.15.83:0/907073394":"2024-04-08T14:36:48.829922+0000","172.21.15.83:6800/2425895246":"2024-04-08T14:36:48.829922+0000","172.21.15.83:0/482476659":"2024-04-08T14:36:48.829922+0000","172.21.15.83:0/1572934944":"2024-04-08T14:36:48.829922+0000","172.21.15.83:0/848305012":"2024-04-08T14:32:22.035463+0000","172.21.15.83:6801/2425895246":"2024-04-08T14:36:48.829922+0000","172.21.15.83:6800/3303842870":"2024-04-08T14:32:22.035463+0000","172.21.15.83:0/3248703110":"2024-04-08T14:32:22.035463+0000","172.21.15.83:0/1472449846":"2024-04-08T14:31:52.613300+0000","172.21.15.83:6800/533197939":"2024-04-08T14:31:52.613300+0000","172.21.15.83:6801/3303842870":"2024-04-08T14:32:22.035463+0000","172.21.15.83:6801/533197939":"2024-04-08T14:31:52.613300+0000","172.21.15.83:0/145418573":"2024-04-08T14:32:22.035463+0000","172.21.15.83:0/1879750037":"2024-04-08T14:31:52.613300+0000","172.21.15.83:0/1319892688":"2024-04-08T14:31:52.613300+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-07T14:37:11.708 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:11 smithi189 systemd[1]: Starting Ceph node-exporter.b for 7388281a-f4eb-11ee-b649-cb9ed24678a4... 2024-04-07T14:37:11.779 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:37:11 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:37:11] "GET /metrics HTTP/1.1" 200 191515 "" "Prometheus/2.33.4" 2024-04-07T14:37:12.165 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:11 smithi189 ceph-mon[25797]: Deploying daemon node-exporter.b on smithi189 2024-04-07T14:37:12.165 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:11 smithi189 ceph-mon[25797]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:12.165 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:11 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/2793677404' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-07T14:37:12.165 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:11 smithi189 bash[42896]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2024-04-07T14:37:12.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:11 smithi083 ceph-mon[24810]: Deploying daemon node-exporter.b on smithi189 2024-04-07T14:37:12.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:11 smithi083 ceph-mon[24810]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:12.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:11 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/2793677404' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-07T14:37:12.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:11 smithi083 ceph-mon[29774]: Deploying daemon node-exporter.b on smithi189 2024-04-07T14:37:12.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:11 smithi083 ceph-mon[29774]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:12.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:11 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/2793677404' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-07T14:37:12.455 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-07T14:37:12.456 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-07T14:37:12.458 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:12 smithi189 bash[42896]: Getting image source signatures 2024-04-07T14:37:12.458 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:12 smithi189 bash[42896]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2024-04-07T14:37:12.458 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:12 smithi189 bash[42896]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2024-04-07T14:37:12.458 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:12 smithi189 bash[42896]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2024-04-07T14:37:12.732 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-07T14:37:13.949 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:13 smithi189 bash[42896]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-04-07T14:37:13.950 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:13 smithi189 bash[42896]: Writing manifest to image destination 2024-04-07T14:37:14.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:13 smithi083 ceph-mon[24810]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:14.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:13 smithi083 ceph-mon[29774]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:14.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:13 smithi189 ceph-mon[25797]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:14.209 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:13 smithi189 podman[42896]: 2024-04-07 14:37:13.890557719 +0000 UTC m=+2.248242779 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-04-07T14:37:14.209 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:13 smithi189 podman[42896]: 2024-04-07 14:37:13.989457348 +0000 UTC m=+2.347142451 container create f7de611efab077414f319cd654b3c6f17fa87db6c7e3f89e7dbccf61da32b043 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-07T14:37:14.209 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 podman[42896]: 2024-04-07 14:37:14.177560944 +0000 UTC m=+2.535246005 container init f7de611efab077414f319cd654b3c6f17fa87db6c7e3f89e7dbccf61da32b043 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-07T14:37:14.209 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 podman[42896]: 2024-04-07 14:37:14.180615295 +0000 UTC m=+2.538300364 container start f7de611efab077414f319cd654b3c6f17fa87db6c7e3f89e7dbccf61da32b043 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-07T14:37:14.209 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.184Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-04-07T14:37:14.209 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.184Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-04-07T14:37:14.209 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.185Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-04-07T14:37:14.209 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.185Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-07T14:37:14.209 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-04-07T14:37:14.209 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=arp 2024-04-07T14:37:14.209 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=bcache 2024-04-07T14:37:14.209 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=bonding 2024-04-07T14:37:14.210 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=btrfs 2024-04-07T14:37:14.210 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=conntrack 2024-04-07T14:37:14.210 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=cpu 2024-04-07T14:37:14.210 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-04-07T14:37:14.210 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=diskstats 2024-04-07T14:37:14.210 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=dmi 2024-04-07T14:37:14.210 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=edac 2024-04-07T14:37:14.210 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=entropy 2024-04-07T14:37:14.210 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-04-07T14:37:14.210 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=filefd 2024-04-07T14:37:14.210 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=filesystem 2024-04-07T14:37:14.210 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=hwmon 2024-04-07T14:37:14.210 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=infiniband 2024-04-07T14:37:14.211 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=ipvs 2024-04-07T14:37:14.211 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=loadavg 2024-04-07T14:37:14.211 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=mdadm 2024-04-07T14:37:14.211 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=meminfo 2024-04-07T14:37:14.211 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=netclass 2024-04-07T14:37:14.211 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=netdev 2024-04-07T14:37:14.211 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=netstat 2024-04-07T14:37:14.211 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=nfs 2024-04-07T14:37:14.211 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=nfsd 2024-04-07T14:37:14.211 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=nvme 2024-04-07T14:37:14.211 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=os 2024-04-07T14:37:14.211 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-04-07T14:37:14.211 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=pressure 2024-04-07T14:37:14.211 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=rapl 2024-04-07T14:37:14.212 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=schedstat 2024-04-07T14:37:14.212 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=sockstat 2024-04-07T14:37:14.212 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=softnet 2024-04-07T14:37:14.212 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=stat 2024-04-07T14:37:14.212 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=tapestats 2024-04-07T14:37:14.212 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=textfile 2024-04-07T14:37:14.212 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-04-07T14:37:14.212 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=time 2024-04-07T14:37:14.212 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-04-07T14:37:14.212 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=uname 2024-04-07T14:37:14.213 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=vmstat 2024-04-07T14:37:14.213 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=xfs 2024-04-07T14:37:14.213 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:115 level=info collector=zfs 2024-04-07T14:37:14.213 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-04-07T14:37:14.213 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b[42947]: ts=2024-04-07T14:37:14.186Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-04-07T14:37:14.703 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 bash[42896]: f7de611efab077414f319cd654b3c6f17fa87db6c7e3f89e7dbccf61da32b043 2024-04-07T14:37:14.703 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:37:14 smithi189 systemd[1]: Started Ceph node-exporter.b for 7388281a-f4eb-11ee-b649-cb9ed24678a4. 2024-04-07T14:37:15.492 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-07T14:37:15.492 INFO:teuthology.orchestra.run.smithi083.stdout:{"epoch":60,"fsid":"7388281a-f4eb-11ee-b649-cb9ed24678a4","created":"2024-04-07T14:31:30.809811+0000","modified":"2024-04-07T14:36:48.829956+0000","last_up_change":"2024-04-07T14:36:28.289077+0000","last_in_change":"2024-04-07T14:36:10.041386+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-07T14:34:36.411287+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"a42cc075-c4af-4500-abae-738cfd1c8d97","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6802","nonce":159265472},{"type":"v1","addr":"172.21.15.83:6803","nonce":159265472}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6804","nonce":159265472},{"type":"v1","addr":"172.21.15.83:6805","nonce":159265472}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6808","nonce":159265472},{"type":"v1","addr":"172.21.15.83:6809","nonce":159265472}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6806","nonce":159265472},{"type":"v1","addr":"172.21.15.83:6807","nonce":159265472}]},"public_addr":"172.21.15.83:6803/159265472","cluster_addr":"172.21.15.83:6805/159265472","heartbeat_back_addr":"172.21.15.83:6809/159265472","heartbeat_front_addr":"172.21.15.83:6807/159265472","state":["exists","up"]},{"osd":1,"uuid":"c0b76e30-ff29-43b7-a8d1-90f9bef15b7e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6810","nonce":4165704838},{"type":"v1","addr":"172.21.15.83:6811","nonce":4165704838}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6812","nonce":4165704838},{"type":"v1","addr":"172.21.15.83:6813","nonce":4165704838}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6816","nonce":4165704838},{"type":"v1","addr":"172.21.15.83:6817","nonce":4165704838}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6814","nonce":4165704838},{"type":"v1","addr":"172.21.15.83:6815","nonce":4165704838}]},"public_addr":"172.21.15.83:6811/4165704838","cluster_addr":"172.21.15.83:6813/4165704838","heartbeat_back_addr":"172.21.15.83:6817/4165704838","heartbeat_front_addr":"172.21.15.83:6815/4165704838","state":["exists","up"]},{"osd":2,"uuid":"fe1ee70e-866b-463e-b7fd-2a2057f44b80","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6818","nonce":718005878},{"type":"v1","addr":"172.21.15.83:6819","nonce":718005878}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6820","nonce":718005878},{"type":"v1","addr":"172.21.15.83:6821","nonce":718005878}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6824","nonce":718005878},{"type":"v1","addr":"172.21.15.83:6825","nonce":718005878}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6822","nonce":718005878},{"type":"v1","addr":"172.21.15.83:6823","nonce":718005878}]},"public_addr":"172.21.15.83:6819/718005878","cluster_addr":"172.21.15.83:6821/718005878","heartbeat_back_addr":"172.21.15.83:6825/718005878","heartbeat_front_addr":"172.21.15.83:6823/718005878","state":["exists","up"]},{"osd":3,"uuid":"14bfed5e-6a77-4448-855b-258e622311ee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6826","nonce":3079630945},{"type":"v1","addr":"172.21.15.83:6827","nonce":3079630945}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6828","nonce":3079630945},{"type":"v1","addr":"172.21.15.83:6829","nonce":3079630945}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6832","nonce":3079630945},{"type":"v1","addr":"172.21.15.83:6833","nonce":3079630945}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.83:6830","nonce":3079630945},{"type":"v1","addr":"172.21.15.83:6831","nonce":3079630945}]},"public_addr":"172.21.15.83:6827/3079630945","cluster_addr":"172.21.15.83:6829/3079630945","heartbeat_back_addr":"172.21.15.83:6833/3079630945","heartbeat_front_addr":"172.21.15.83:6831/3079630945","state":["exists","up"]},{"osd":4,"uuid":"6a8687be-8c65-40a3-89c4-d463ed60d920","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6800","nonce":3355623828},{"type":"v1","addr":"172.21.15.189:6801","nonce":3355623828}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6802","nonce":3355623828},{"type":"v1","addr":"172.21.15.189:6803","nonce":3355623828}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6806","nonce":3355623828},{"type":"v1","addr":"172.21.15.189:6807","nonce":3355623828}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6804","nonce":3355623828},{"type":"v1","addr":"172.21.15.189:6805","nonce":3355623828}]},"public_addr":"172.21.15.189:6801/3355623828","cluster_addr":"172.21.15.189:6803/3355623828","heartbeat_back_addr":"172.21.15.189:6807/3355623828","heartbeat_front_addr":"172.21.15.189:6805/3355623828","state":["exists","up"]},{"osd":5,"uuid":"7115ab07-1339-4eac-b403-bf6b40361af9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6808","nonce":3207731440},{"type":"v1","addr":"172.21.15.189:6809","nonce":3207731440}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6810","nonce":3207731440},{"type":"v1","addr":"172.21.15.189:6811","nonce":3207731440}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6814","nonce":3207731440},{"type":"v1","addr":"172.21.15.189:6815","nonce":3207731440}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6812","nonce":3207731440},{"type":"v1","addr":"172.21.15.189:6813","nonce":3207731440}]},"public_addr":"172.21.15.189:6809/3207731440","cluster_addr":"172.21.15.189:6811/3207731440","heartbeat_back_addr":"172.21.15.189:6815/3207731440","heartbeat_front_addr":"172.21.15.189:6813/3207731440","state":["exists","up"]},{"osd":6,"uuid":"1411a622-2b3f-48f8-b265-8b58643679f0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6816","nonce":3839485321},{"type":"v1","addr":"172.21.15.189:6817","nonce":3839485321}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6818","nonce":3839485321},{"type":"v1","addr":"172.21.15.189:6819","nonce":3839485321}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6822","nonce":3839485321},{"type":"v1","addr":"172.21.15.189:6823","nonce":3839485321}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6820","nonce":3839485321},{"type":"v1","addr":"172.21.15.189:6821","nonce":3839485321}]},"public_addr":"172.21.15.189:6817/3839485321","cluster_addr":"172.21.15.189:6819/3839485321","heartbeat_back_addr":"172.21.15.189:6823/3839485321","heartbeat_front_addr":"172.21.15.189:6821/3839485321","state":["exists","up"]},{"osd":7,"uuid":"bbaaa4cf-7ff3-4a75-aeb8-d8325b9313c1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6824","nonce":764170876},{"type":"v1","addr":"172.21.15.189:6825","nonce":764170876}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6826","nonce":764170876},{"type":"v1","addr":"172.21.15.189:6827","nonce":764170876}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6830","nonce":764170876},{"type":"v1","addr":"172.21.15.189:6831","nonce":764170876}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.189:6828","nonce":764170876},{"type":"v1","addr":"172.21.15.189:6829","nonce":764170876}]},"public_addr":"172.21.15.189:6825/764170876","cluster_addr":"172.21.15.189:6827/764170876","heartbeat_back_addr":"172.21.15.189:6831/764170876","heartbeat_front_addr":"172.21.15.189:6829/764170876","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-07T14:33:44.598937+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-07T14:34:07.818529+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-07T14:34:32.826953+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-07T14:34:56.635361+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-07T14:35:18.427639+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-07T14:35:40.181583+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-07T14:36:02.709930+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-07T14:36:26.467881+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.83:0/248923343":"2024-04-08T14:36:48.829922+0000","172.21.15.83:0/907073394":"2024-04-08T14:36:48.829922+0000","172.21.15.83:6800/2425895246":"2024-04-08T14:36:48.829922+0000","172.21.15.83:0/482476659":"2024-04-08T14:36:48.829922+0000","172.21.15.83:0/1572934944":"2024-04-08T14:36:48.829922+0000","172.21.15.83:0/848305012":"2024-04-08T14:32:22.035463+0000","172.21.15.83:6801/2425895246":"2024-04-08T14:36:48.829922+0000","172.21.15.83:6800/3303842870":"2024-04-08T14:32:22.035463+0000","172.21.15.83:0/3248703110":"2024-04-08T14:32:22.035463+0000","172.21.15.83:0/1472449846":"2024-04-08T14:31:52.613300+0000","172.21.15.83:6800/533197939":"2024-04-08T14:31:52.613300+0000","172.21.15.83:6801/3303842870":"2024-04-08T14:32:22.035463+0000","172.21.15.83:6801/533197939":"2024-04-08T14:31:52.613300+0000","172.21.15.83:0/145418573":"2024-04-08T14:32:22.035463+0000","172.21.15.83:0/1879750037":"2024-04-08T14:31:52.613300+0000","172.21.15.83:0/1319892688":"2024-04-08T14:31:52.613300+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-07T14:37:15.493 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:15 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:15.493 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:15 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:37:15.493 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:15 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:37:15.493 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:15 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:37:15.493 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:15 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:15.493 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:15 smithi083 ceph-mon[24810]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:37:15.493 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:15 smithi083 ceph-mon[24810]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-07T14:37:15.493 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:15 smithi083 ceph-mon[24810]: Reconfiguring daemon prometheus.a on smithi189 2024-04-07T14:37:15.493 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:15 smithi083 ceph-mon[24810]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:15.493 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:15 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:15.494 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:15 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:37:15.494 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:15 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:37:15.494 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:15 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:37:15.494 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:15 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:15.494 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:15 smithi083 ceph-mon[29774]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:37:15.494 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:15 smithi083 ceph-mon[29774]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-07T14:37:15.494 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:15 smithi083 ceph-mon[29774]: Reconfiguring daemon prometheus.a on smithi189 2024-04-07T14:37:15.494 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:15 smithi083 ceph-mon[29774]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:15.667 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:15 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:15.667 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:15 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:37:15.667 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:15 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:37:15.667 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:15 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:37:15.667 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:15 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:15.667 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:15 smithi189 ceph-mon[25797]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:37:15.667 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:15 smithi189 ceph-mon[25797]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-07T14:37:15.667 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:15 smithi189 ceph-mon[25797]: Reconfiguring daemon prometheus.a on smithi189 2024-04-07T14:37:15.668 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:15 smithi189 ceph-mon[25797]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:15.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:37:15 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:37:15] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:37:16.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:16 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/979053852' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-07T14:37:16.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:16 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:16.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:16 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/979053852' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-07T14:37:16.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:16 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:16.597 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:16 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/979053852' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-07T14:37:16.597 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:16 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:17.137 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph tell osd.0 flush_pg_stats 2024-04-07T14:37:17.137 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph tell osd.1 flush_pg_stats 2024-04-07T14:37:17.138 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph tell osd.2 flush_pg_stats 2024-04-07T14:37:17.138 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph tell osd.3 flush_pg_stats 2024-04-07T14:37:17.139 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph tell osd.4 flush_pg_stats 2024-04-07T14:37:17.139 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph tell osd.5 flush_pg_stats 2024-04-07T14:37:17.139 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph tell osd.6 flush_pg_stats 2024-04-07T14:37:17.140 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph tell osd.7 flush_pg_stats 2024-04-07T14:37:17.390 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:17 smithi083 ceph-mon[24810]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:17.390 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:17 smithi083 ceph-mon[29774]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:17.560 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:17 smithi189 ceph-mon[25797]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:17.560 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-07T14:37:17.588 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-07T14:37:17.641 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-07T14:37:17.659 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-07T14:37:17.659 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-07T14:37:17.660 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-07T14:37:17.660 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-07T14:37:17.660 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-07T14:37:18.707 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:37:18 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:37:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:37:18.708 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:18 smithi189 systemd[1]: Stopping Ceph prometheus.a for 7388281a-f4eb-11ee-b649-cb9ed24678a4... 2024-04-07T14:37:18.708 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:18 smithi189 bash[43276]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus.a" found: no such container 2024-04-07T14:37:18.958 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:18 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[42032]: ts=2024-04-07T14:37:18.937Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-07T14:37:18.959 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:18 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[42032]: ts=2024-04-07T14:37:18.937Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-07T14:37:18.959 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:18 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[42032]: ts=2024-04-07T14:37:18.937Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-07T14:37:18.959 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:18 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[42032]: ts=2024-04-07T14:37:18.937Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-07T14:37:18.959 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:18 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[42032]: ts=2024-04-07T14:37:18.937Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-07T14:37:18.959 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:18 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[42032]: ts=2024-04-07T14:37:18.937Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-07T14:37:18.959 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:18 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[42032]: ts=2024-04-07T14:37:18.937Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-07T14:37:18.959 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:18 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[42032]: ts=2024-04-07T14:37:18.937Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-07T14:37:18.959 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:18 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[42032]: ts=2024-04-07T14:37:18.937Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-07T14:37:18.959 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:18 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[42032]: ts=2024-04-07T14:37:18.938Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-07T14:37:18.959 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:18 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[42032]: ts=2024-04-07T14:37:18.938Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-07T14:37:18.959 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:18 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[42032]: ts=2024-04-07T14:37:18.938Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-07T14:37:18.959 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:18 smithi189 podman[43297]: 2024-04-07 14:37:18.947969894 +0000 UTC m=+0.454308657 container died ac215bd79e01b73d669acb41c2a26d86c09ec88e71db0e579c34732d9048052a (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-07T14:37:19.470 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:19 smithi189 podman[43297]: 2024-04-07 14:37:19.31248907 +0000 UTC m=+0.818827837 container cleanup ac215bd79e01b73d669acb41c2a26d86c09ec88e71db0e579c34732d9048052a (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-07T14:37:19.470 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:19 smithi189 bash[43297]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a 2024-04-07T14:37:19.470 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:19 smithi189 bash[43348]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus.a" found: no such container 2024-04-07T14:37:20.147 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:19 smithi189 ceph-mon[25797]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:20.148 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:19 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:20.148 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:19 smithi189 podman[43336]: 2024-04-07 14:37:19.8819666 +0000 UTC m=+0.933258317 container remove ac215bd79e01b73d669acb41c2a26d86c09ec88e71db0e579c34732d9048052a (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-07T14:37:20.148 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:19 smithi189 systemd[1]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@prometheus.a.service: Deactivated successfully. 2024-04-07T14:37:20.148 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:19 smithi189 systemd[1]: Stopped Ceph prometheus.a for 7388281a-f4eb-11ee-b649-cb9ed24678a4. 2024-04-07T14:37:20.148 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:19 smithi189 systemd[1]: Starting Ceph prometheus.a for 7388281a-f4eb-11ee-b649-cb9ed24678a4... 2024-04-07T14:37:20.148 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:20 smithi189 podman[43476]: 2024-04-07 14:37:20.034609211 +0000 UTC m=+0.027008678 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-07T14:37:20.148 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:20 smithi189 podman[43476]: 2024-04-07 14:37:20.149525062 +0000 UTC m=+0.141924509 container create 19bf72cd5f3b217072f4aed2dd6c46aeb0ec85f6dfda16b2d1175a2b489a9dce (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-07T14:37:20.350 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:19 smithi083 ceph-mon[24810]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:20.350 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:19 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:20.350 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:19 smithi083 ceph-mon[29774]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:20.350 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:19 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:20.420 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:20 smithi189 podman[43476]: 2024-04-07 14:37:20.395679551 +0000 UTC m=+0.388078999 container init 19bf72cd5f3b217072f4aed2dd6c46aeb0ec85f6dfda16b2d1175a2b489a9dce (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-07T14:37:20.420 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:20 smithi189 podman[43476]: 2024-04-07 14:37:20.398345385 +0000 UTC m=+0.390744836 container start 19bf72cd5f3b217072f4aed2dd6c46aeb0ec85f6dfda16b2d1175a2b489a9dce (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-07T14:37:20.709 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:20 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[43522]: ts=2024-04-07T14:37:20.422Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-07T14:37:20.709 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:20 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[43522]: ts=2024-04-07T14:37:20.422Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-07T14:37:20.709 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:20 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[43522]: ts=2024-04-07T14:37:20.422Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-07T14:37:20.710 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:20 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[43522]: ts=2024-04-07T14:37:20.422Z caller=main.go:518 level=info host_details="(Linux 5.14.0-432.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Wed Mar 20 20:26:59 UTC 2024 x86_64 smithi189 (none))" 2024-04-07T14:37:20.710 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:20 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[43522]: ts=2024-04-07T14:37:20.422Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-07T14:37:20.710 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:20 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[43522]: ts=2024-04-07T14:37:20.422Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-07T14:37:20.710 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:20 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[43522]: ts=2024-04-07T14:37:20.424Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-07T14:37:20.710 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:20 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[43522]: ts=2024-04-07T14:37:20.424Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-07T14:37:20.710 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:20 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[43522]: ts=2024-04-07T14:37:20.425Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-07T14:37:20.710 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:20 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[43522]: ts=2024-04-07T14:37:20.428Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-07T14:37:20.710 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:20 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[43522]: ts=2024-04-07T14:37:20.428Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.632µs 2024-04-07T14:37:20.711 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:20 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[43522]: ts=2024-04-07T14:37:20.428Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-07T14:37:20.711 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:20 smithi189 bash[43476]: 19bf72cd5f3b217072f4aed2dd6c46aeb0ec85f6dfda16b2d1175a2b489a9dce 2024-04-07T14:37:20.711 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:20 smithi189 systemd[1]: Started Ceph prometheus.a for 7388281a-f4eb-11ee-b649-cb9ed24678a4. 2024-04-07T14:37:20.711 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:20 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[43522]: ts=2024-04-07T14:37:20.651Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-04-07T14:37:20.711 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:20 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[43522]: ts=2024-04-07T14:37:20.651Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-04-07T14:37:20.711 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:20 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[43522]: ts=2024-04-07T14:37:20.651Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=24.957µs wal_replay_duration=223.212325ms total_replay_duration=223.251026ms 2024-04-07T14:37:20.711 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:20 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[43522]: ts=2024-04-07T14:37:20.653Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-07T14:37:20.711 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:20 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[43522]: ts=2024-04-07T14:37:20.653Z caller=main.go:947 level=info msg="TSDB started" 2024-04-07T14:37:20.712 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:20 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[43522]: ts=2024-04-07T14:37:20.653Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-07T14:37:20.712 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:20 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[43522]: ts=2024-04-07T14:37:20.669Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=16.279903ms db_storage=632ns remote_storage=1.192µs web_handler=333ns query_engine=711ns scrape=362.569µs scrape_sd=23.247µs notify=850ns notify_sd=1.202µs rules=15.680846ms 2024-04-07T14:37:20.712 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:37:20 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[43522]: ts=2024-04-07T14:37:20.669Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-07T14:37:21.469 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:37:21 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:37:21] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:37:21.803 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:21 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:21.803 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:21 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-07T14:37:21.803 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:21 smithi189 ceph-mon[25797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-07T14:37:21.803 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:21 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.189:9095"}]: dispatch 2024-04-07T14:37:21.803 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:21 smithi189 ceph-mon[25797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.189:9095"}]: dispatch 2024-04-07T14:37:21.803 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:21 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:21.803 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:21 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:37:21.803 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:21 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:37:21.804 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:21 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:37:21.804 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:21 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:21.804 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:21 smithi189 ceph-mon[25797]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:37:21.804 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:21 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:37:21.804 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:21 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:37:21.804 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:21 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:37:21.804 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:21 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:21.804 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:21 smithi189 ceph-mon[25797]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:37:21.804 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:21 smithi189 ceph-mon[25797]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:21.804 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:21 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:21.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:21 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:21.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:21 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-07T14:37:21.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:21 smithi083 ceph-mon[24810]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-07T14:37:21.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:21 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.189:9095"}]: dispatch 2024-04-07T14:37:21.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:21 smithi083 ceph-mon[24810]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.189:9095"}]: dispatch 2024-04-07T14:37:21.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:21 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:21.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:21 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:37:21.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:21 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:37:21.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:21 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:37:21.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:21 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:21.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:21 smithi083 ceph-mon[24810]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:37:21.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:21 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:37:21.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:21 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:37:21.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:21 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:37:21.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:21 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:21.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:21 smithi083 ceph-mon[24810]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:37:21.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:21 smithi083 ceph-mon[24810]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:21.927 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:21 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:21.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:21 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:21.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:21 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-07T14:37:21.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:21 smithi083 ceph-mon[29774]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-07T14:37:21.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:21 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.189:9095"}]: dispatch 2024-04-07T14:37:21.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:21 smithi083 ceph-mon[29774]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.189:9095"}]: dispatch 2024-04-07T14:37:21.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:21 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:21.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:21 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:37:21.928 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:21 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:37:21.928 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:21 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:37:21.928 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:21 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:21.928 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:21 smithi083 ceph-mon[29774]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:37:21.928 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:21 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:37:21.928 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:21 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:37:21.928 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:21 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:37:21.928 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:21 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:21.928 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:21 smithi083 ceph-mon[29774]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:37:21.928 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:21 smithi083 ceph-mon[29774]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:21.928 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:21 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:22.396 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:37:22 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:37:22] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:37:22.676 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:37:22 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:37:22] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:37:23.044 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:23 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:23.044 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:23 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:23.044 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:23 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:23.044 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:23 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:37:23.044 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:23 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:37:23.044 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:23 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:37:23.044 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:23 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:23.425 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:37:23 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:37:23] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:37:23.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:23 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:23.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:23 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:23.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:23 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:23.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:23 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:37:23.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:23 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:37:23.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:23 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:37:23.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:23 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:23.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:23 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:23.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:23 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:23.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:23 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:23.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:23 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:37:23.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:23 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:37:23.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:23 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:37:23.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:23 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:24.141 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:24 smithi083 ceph-mon[24810]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:24.141 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:24 smithi083 ceph-mon[24810]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:24.142 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:24 smithi083 ceph-mon[24810]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:24.142 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:24 smithi083 ceph-mon[24810]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:37:24.142 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:24 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:24.142 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:24 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:24.142 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:24 smithi083 ceph-mon[24810]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-07T14:37:24.143 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:24 smithi083 ceph-mon[29774]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:24.143 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:24 smithi083 ceph-mon[29774]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:24.143 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:24 smithi083 ceph-mon[29774]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:24.143 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:24 smithi083 ceph-mon[29774]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:37:24.143 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:24 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:24.143 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:24 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:24.143 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:24 smithi083 ceph-mon[29774]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-07T14:37:24.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:24 smithi189 ceph-mon[25797]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:24.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:24 smithi189 ceph-mon[25797]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:24.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:24 smithi189 ceph-mon[25797]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:24.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:24 smithi189 ceph-mon[25797]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:37:24.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:24 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:24.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:24 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:24.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:24 smithi189 ceph-mon[25797]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-07T14:37:25.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:25 smithi083 ceph-mon[24810]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:25.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:25 smithi083 ceph-mon[24810]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:25.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:25 smithi083 ceph-mon[29774]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:25.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:25 smithi083 ceph-mon[29774]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:25.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:25 smithi189 ceph-mon[25797]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:25.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:25 smithi189 ceph-mon[25797]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:27.338 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:27 smithi083 ceph-mon[24810]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:27.338 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:27 smithi083 ceph-mon[29774]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:27.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:27 smithi189 ceph-mon[25797]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:28.151 INFO:teuthology.orchestra.run.smithi083.stdout:34359738415 2024-04-07T14:37:28.152 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.0 2024-04-07T14:37:28.708 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:37:28 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:37:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:37:29.322 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:29 smithi083 ceph-mon[24810]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:29.322 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:29 smithi083 ceph-mon[29774]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:29.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:29 smithi189 ceph-mon[25797]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:29.954 INFO:teuthology.orchestra.run.smithi083.stdout:236223201294 2024-04-07T14:37:29.954 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.7 2024-04-07T14:37:31.285 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:37:31 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:37:31] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:37:31.286 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:31 smithi083 ceph-mon[24810]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:31.286 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:31 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:31.286 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:31 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:31.286 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:31 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:37:31.286 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:31 smithi083 ceph-mon[29774]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:31.286 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:31 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:31.286 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:31 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:31.286 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:31 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:37:31.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:31 smithi189 ceph-mon[25797]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:31.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:31 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:31.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:31 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:31.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:31 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-07T14:37:31.926 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:37:31 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:37:31] "GET /metrics HTTP/1.1" 200 191498 "" "Prometheus/2.33.4" 2024-04-07T14:37:32.224 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:32 smithi083 ceph-mon[24810]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:32.224 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:32 smithi083 ceph-mon[24810]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:32.224 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:32 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:37:32.225 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:32 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:37:32.225 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:32 smithi083 ceph-mon[24810]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:32.225 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:32 smithi083 ceph-mon[24810]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:32.225 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:32 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:32.225 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:32 smithi083 ceph-mon[24810]: Deploying daemon alertmanager.a on smithi083 2024-04-07T14:37:32.225 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:32 smithi083 ceph-mon[29774]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:32.225 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:32 smithi083 ceph-mon[29774]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:32.225 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:32 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:37:32.225 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:32 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:37:32.225 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:32 smithi083 ceph-mon[29774]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:32.225 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:32 smithi083 ceph-mon[29774]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:32.226 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:32 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:32.226 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:32 smithi083 ceph-mon[29774]: Deploying daemon alertmanager.a on smithi083 2024-04-07T14:37:32.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:32 smithi189 ceph-mon[25797]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:32.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:32 smithi189 ceph-mon[25797]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:32.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:32 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:37:32.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:32 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:37:32.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:32 smithi189 ceph-mon[25797]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:32.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:32 smithi189 ceph-mon[25797]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:32.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:32 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:32.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:32 smithi189 ceph-mon[25797]: Deploying daemon alertmanager.a on smithi083 2024-04-07T14:37:32.589 INFO:teuthology.orchestra.run.smithi083.stdout:55834574890 2024-04-07T14:37:32.589 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.1 2024-04-07T14:37:34.035 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-07T14:37:34.035 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-07T14:37:34.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:34 smithi083 ceph-mon[24810]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:34.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:34 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:34.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:34 smithi083 ceph-mon[29774]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:34.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:34 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:34.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:34 smithi189 ceph-mon[25797]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:34.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:34 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:34.670 INFO:teuthology.orchestra.run.smithi083.stdout:133143986204 2024-04-07T14:37:34.671 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.4 2024-04-07T14:37:35.405 INFO:teuthology.orchestra.run.smithi083.stdout:111669149728 2024-04-07T14:37:35.405 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.3 2024-04-07T14:37:36.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:36 smithi083 ceph-mon[24810]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:36.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:36 smithi083 ceph-mon[29774]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:36.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:36 smithi189 ceph-mon[25797]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:36.587 INFO:teuthology.orchestra.run.smithi083.stdout:77309411366 2024-04-07T14:37:36.587 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.2 2024-04-07T14:37:37.354 INFO:teuthology.orchestra.run.smithi083.stdout:201863462931 2024-04-07T14:37:37.355 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.6 2024-04-07T14:37:38.291 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:38 smithi083 ceph-mon[24810]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:38.291 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:38 smithi083 ceph-mon[29774]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:38.404 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:38 smithi189 ceph-mon[25797]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:38.404 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:37:38 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:37:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:37:39.426 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:37:39 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:37:39] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:37:40.097 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-07T14:37:40.098 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-07T14:37:40.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:40 smithi083 ceph-mon[29774]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:40.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:40 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:40.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:40 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:40.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:40 smithi083 ceph-mon[24810]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:40.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:40 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:40.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:40 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:40.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:40 smithi189 ceph-mon[25797]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:40.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:40 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:40.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:40 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:40.577 INFO:teuthology.orchestra.run.smithi083.stdout:34359738417 2024-04-07T14:37:41.031 INFO:teuthology.orchestra.run.smithi083.stdout:167503724569 2024-04-07T14:37:41.032 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.5 2024-04-07T14:37:41.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:41 smithi083 ceph-mon[24810]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:41.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:41 smithi083 ceph-mon[24810]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-07T14:37:41.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:41 smithi083 ceph-mon[24810]: Cluster is now healthy 2024-04-07T14:37:41.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:41 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/3781559067' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-07T14:37:41.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:41 smithi083 ceph-mon[29774]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:41.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:41 smithi083 ceph-mon[29774]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-07T14:37:41.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:41 smithi083 ceph-mon[29774]: Cluster is now healthy 2024-04-07T14:37:41.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:41 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/3781559067' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-07T14:37:41.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:41 smithi189 ceph-mon[25797]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:41.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:41 smithi189 ceph-mon[25797]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-07T14:37:41.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:41 smithi189 ceph-mon[25797]: Cluster is now healthy 2024-04-07T14:37:41.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:41 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/3781559067' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-07T14:37:41.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:37:41 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:37:41] "GET /metrics HTTP/1.1" 200 191560 "" "Prometheus/2.33.4" 2024-04-07T14:37:42.435 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738415 got 34359738417 for osd.0 2024-04-07T14:37:42.436 DEBUG:teuthology.parallel:result is None 2024-04-07T14:37:42.778 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-07T14:37:42.779 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-07T14:37:42.779 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-07T14:37:43.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:43 smithi189 ceph-mon[25797]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:43.413 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:43 smithi083 ceph-mon[24810]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:43.414 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:43 smithi083 ceph-mon[29774]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:45.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:45 smithi083 ceph-mon[24810]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:45.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:45 smithi083 ceph-mon[29774]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:45.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:45 smithi189 ceph-mon[25797]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:45.800 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-07T14:37:46.598 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:46 smithi083 ceph-mon[24810]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:46.599 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:46 smithi083 ceph-mon[29774]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:46.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:46 smithi189 ceph-mon[25797]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:48.535 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:48 smithi083 ceph-mon[24810]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:48.536 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:48 smithi083 ceph-mon[29774]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:48.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:48 smithi189 ceph-mon[25797]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:48.708 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:37:48 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:37:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:37:49.996 INFO:teuthology.orchestra.run.smithi083.stdout:236223201298 2024-04-07T14:37:50.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:50.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:37:50.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:37:50.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:50.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:37:50.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:37:50.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:50.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:37:50.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:37:51.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:50 smithi189 ceph-mon[25797]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:51.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:50 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/64309636' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-07T14:37:51.426 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:37:51 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:37:51] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:37:51.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:50 smithi083 ceph-mon[29774]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:51.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:50 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/64309636' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-07T14:37:51.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:50 smithi083 ceph-mon[24810]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:51.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:50 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/64309636' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-07T14:37:51.757 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:37:51 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:37:51] "GET /metrics HTTP/1.1" 200 191573 "" "Prometheus/2.33.4" 2024-04-07T14:37:52.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:52 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:52.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:52 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:52.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:52 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:52.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:52 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:52.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:52 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:52.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:52 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:37:52.547 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201294 got 236223201298 for osd.7 2024-04-07T14:37:52.548 DEBUG:teuthology.parallel:result is None 2024-04-07T14:37:53.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:53 smithi083 ceph-mon[24810]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:53.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:53 smithi083 ceph-mon[29774]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:53.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:53 smithi189 ceph-mon[25797]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:54.422 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:54 smithi083 ceph-mon[24810]: pgmap v43: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:54.422 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:54 smithi083 ceph-mon[29774]: pgmap v43: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:54.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:54 smithi189 ceph-mon[25797]: pgmap v43: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:54.620 INFO:teuthology.orchestra.run.smithi083.stdout:133143986208 2024-04-07T14:37:54.814 INFO:teuthology.orchestra.run.smithi083.stdout:201863462935 2024-04-07T14:37:55.025 INFO:teuthology.orchestra.run.smithi083.stdout:167503724572 2024-04-07T14:37:55.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:55 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/4062559636' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-07T14:37:55.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:55 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/104556022' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-07T14:37:55.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:55 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/3137725917' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-07T14:37:55.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:55 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/4062559636' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-07T14:37:55.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:55 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/104556022' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-07T14:37:55.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:55 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/3137725917' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-07T14:37:55.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:55 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/4062559636' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-07T14:37:55.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:55 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/104556022' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-07T14:37:55.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:55 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/3137725917' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-07T14:37:56.290 INFO:teuthology.orchestra.run.smithi083.stdout:77309411371 2024-04-07T14:37:56.636 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:56 smithi083 ceph-mon[24810]: pgmap v44: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:56.636 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:56 smithi083 ceph-mon[29774]: pgmap v44: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:56.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:56 smithi189 ceph-mon[25797]: pgmap v44: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:57.298 INFO:teuthology.orchestra.run.smithi083.stdout:111669149733 2024-04-07T14:37:57.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:57 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/1710597504' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-07T14:37:57.528 INFO:teuthology.orchestra.run.smithi083.stdout:55834574896 2024-04-07T14:37:57.529 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:57 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/1710597504' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-07T14:37:57.529 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:57 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/1710597504' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-07T14:37:58.229 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574890 got 55834574896 for osd.1 2024-04-07T14:37:58.229 DEBUG:teuthology.parallel:result is None 2024-04-07T14:37:58.643 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:58 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/310330130' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-07T14:37:58.643 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:58 smithi083 ceph-mon[24810]: pgmap v45: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:58.643 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:37:58 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/512534385' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-07T14:37:58.643 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:58 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/310330130' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-07T14:37:58.644 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:58 smithi083 ceph-mon[29774]: pgmap v45: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:58.644 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:37:58 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/512534385' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-07T14:37:58.668 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462931 got 201863462935 for osd.6 2024-04-07T14:37:58.668 DEBUG:teuthology.parallel:result is None 2024-04-07T14:37:58.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:58 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/310330130' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-07T14:37:58.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:58 smithi189 ceph-mon[25797]: pgmap v45: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:37:58.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:37:58 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/512534385' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-07T14:37:58.709 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:37:58 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:37:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:37:59.834 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:37:59 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:37:59] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:38:00.507 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411366 got 77309411371 for osd.2 2024-04-07T14:38:00.507 DEBUG:teuthology.parallel:result is None 2024-04-07T14:38:00.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:00 smithi189 ceph-mon[25797]: pgmap v46: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:38:00.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:00 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:00.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:00 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:00.914 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:00 smithi083 ceph-mon[24810]: pgmap v46: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:38:00.915 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:00 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:00.915 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:00 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:00.915 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:00 smithi083 ceph-mon[29774]: pgmap v46: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:38:00.915 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:00 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:00.915 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:00 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:00.943 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986204 got 133143986208 for osd.4 2024-04-07T14:38:00.943 DEBUG:teuthology.parallel:result is None 2024-04-07T14:38:01.360 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149728 got 111669149733 for osd.3 2024-04-07T14:38:01.360 DEBUG:teuthology.parallel:result is None 2024-04-07T14:38:01.693 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724569 got 167503724572 for osd.5 2024-04-07T14:38:01.693 DEBUG:teuthology.parallel:result is None 2024-04-07T14:38:01.693 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-07T14:38:01.694 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph pg dump --format=json 2024-04-07T14:38:01.926 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:38:01 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:38:01] "GET /metrics HTTP/1.1" 200 191573 "" "Prometheus/2.33.4" 2024-04-07T14:38:02.257 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-07T14:38:02.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:02 smithi083 ceph-mon[24810]: pgmap v47: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:38:02.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:02 smithi083 ceph-mon[29774]: pgmap v47: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:38:02.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:02 smithi189 ceph-mon[25797]: pgmap v47: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:38:04.138 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:04 smithi083 podman[49417]: 2024-04-07 14:38:04.029557985 +0000 UTC m=+0.917943151 container create f5a5d0d8ddf604e53529accc3e3d3ea56e6fd1eb5ab4c73c83c0dbbc5347bae3 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-07T14:38:04.422 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-07T14:38:04.423 INFO:teuthology.orchestra.run.smithi083.stderr:dumped all 2024-04-07T14:38:04.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:04 smithi083 ceph-mon[24810]: pgmap v48: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:38:04.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:04 smithi083 ceph-mon[29774]: pgmap v48: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:38:04.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:04 smithi189 ceph-mon[25797]: pgmap v48: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:38:04.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:04 smithi083 podman[49417]: 2024-04-07 14:38:04.801782262 +0000 UTC m=+1.690167439 container init f5a5d0d8ddf604e53529accc3e3d3ea56e6fd1eb5ab4c73c83c0dbbc5347bae3 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-07T14:38:04.964 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:04 smithi083 podman[49417]: 2024-04-07 14:38:04.805042351 +0000 UTC m=+1.693427520 container start f5a5d0d8ddf604e53529accc3e3d3ea56e6fd1eb5ab4c73c83c0dbbc5347bae3 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-07T14:38:04.965 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:04 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[49483]: level=info ts=2024-04-07T14:38:04.815Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-04-07T14:38:04.965 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:04 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[49483]: level=info ts=2024-04-07T14:38:04.815Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-04-07T14:38:04.965 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:04 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[49483]: level=info ts=2024-04-07T14:38:04.816Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.83 port=9094 2024-04-07T14:38:04.965 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:04 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[49483]: level=info ts=2024-04-07T14:38:04.817Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-07T14:38:04.965 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:04 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[49483]: level=info ts=2024-04-07T14:38:04.843Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-07T14:38:04.965 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:04 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[49483]: level=info ts=2024-04-07T14:38:04.843Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-07T14:38:04.965 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:04 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[49483]: level=info ts=2024-04-07T14:38:04.846Z caller=main.go:518 msg=Listening address=:9093 2024-04-07T14:38:04.965 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:04 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[49483]: level=info ts=2024-04-07T14:38:04.846Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-04-07T14:38:04.965 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:04 smithi083 bash[49417]: f5a5d0d8ddf604e53529accc3e3d3ea56e6fd1eb5ab4c73c83c0dbbc5347bae3 2024-04-07T14:38:04.965 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:04 smithi083 systemd[1]: Started Ceph alertmanager.a for 7388281a-f4eb-11ee-b649-cb9ed24678a4. 2024-04-07T14:38:05.381 INFO:teuthology.orchestra.run.smithi083.stdout:{"pg_ready":true,"pg_map":{"version":48,"stamp":"2024-04-07T14:38:03.351228+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48900,"kb_used_data":6148,"kb_used_omap":0,"kb_used_meta":42688,"kb_avail":749912316,"statfs":{"total":767960285184,"available":767910211584,"internally_reserved":0,"allocated":6295552,"data_stored":3510644,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43712512},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002285"},"pg_stats":[{"pgid":"1.0","version":"60'128","reported_seq":266,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-04-07T14:36:49.000086+0000","last_change":"2024-04-07T14:36:32.319838+0000","last_active":"2024-04-07T14:36:49.000086+0000","last_peered":"2024-04-07T14:36:49.000086+0000","last_clean":"2024-04-07T14:36:49.000086+0000","last_became_active":"2024-04-07T14:36:32.319439+0000","last_became_peered":"2024-04-07T14:36:32.319439+0000","last_unstale":"2024-04-07T14:36:49.000086+0000","last_undegraded":"2024-04-07T14:36:49.000086+0000","last_fullsized":"2024-04-07T14:36:49.000086+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-07T14:34:36.446853+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-07T14:34:36.446853+0000","last_clean_scrub_stamp":"2024-04-07T14:34:36.446853+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-09T01:45:25.294322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201301,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6092,"kb_used_data":1092,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739060,"statfs":{"total":95995035648,"available":95988797440,"internally_reserved":0,"allocated":1118208,"data_stored":769323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Apr 7 14:37:28 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.54900000000000004}]},{"osd":1,"last update":"Sun Apr 7 14:37:28 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.51500000000000001}]},{"osd":2,"last update":"Sun Apr 7 14:37:28 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.5}]},{"osd":3,"last update":"Sun Apr 7 14:37:28 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.52300000000000002}]},{"osd":4,"last update":"Sun Apr 7 14:37:28 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.40000000000000002}]},{"osd":5,"last update":"Sun Apr 7 14:37:28 2024","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.377},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.38800000000000001}]},{"osd":6,"last update":"Sun Apr 7 14:37:28 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.36499999999999999}]}]},{"osd":6,"up_from":47,"seq":201863462937,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6152,"kb_used_data":1088,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739000,"statfs":{"total":95995035648,"available":95988736000,"internally_reserved":0,"allocated":1114112,"data_stored":769000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Apr 7 14:37:05 2024","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.442}]},{"osd":1,"last update":"Sun Apr 7 14:37:05 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.35899999999999999}]},{"osd":2,"last update":"Sun Apr 7 14:37:05 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.47699999999999998}]},{"osd":3,"last update":"Sun Apr 7 14:37:05 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.38},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.49099999999999999}]},{"osd":4,"last update":"Sun Apr 7 14:37:05 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.443,"5min":0.443,"15min":0.443},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.51000000000000001}]},{"osd":5,"last update":"Sun Apr 7 14:37:05 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.435}]},{"osd":7,"last update":"Sun Apr 7 14:37:30 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.45600000000000002}]}]},{"osd":1,"up_from":13,"seq":55834574897,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6340,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738812,"statfs":{"total":95995035648,"available":95988543488,"internally_reserved":0,"allocated":585728,"data_stored":240416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Apr 7 14:37:12 2024","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.35099999999999998,"15min":0.34499999999999997},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.64100000000000001,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.36399999999999999,"15min":0.34799999999999998},"min":{"1min":0.26700000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.27600000000000002}]},{"osd":2,"last update":"Sun Apr 7 14:37:41 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.39500000000000002,"15min":0.39300000000000002},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.44700000000000001,"15min":0.441},"min":{"1min":0.311,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.35499999999999998}]},{"osd":3,"last update":"Sun Apr 7 14:37:03 2024","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.42899999999999999,"15min":0.432},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.55700000000000005,"5min":0.59199999999999997,"15min":0.59199999999999997},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.438,"15min":0.442},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.622,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.42399999999999999}]},{"osd":4,"last update":"Sun Apr 7 14:37:22 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.54700000000000004,"15min":0.54500000000000004},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.77100000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.55600000000000005,"15min":0.55000000000000004},"min":{"1min":0.38500000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.49399999999999999}]},{"osd":5,"last update":"Sun Apr 7 14:37:49 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.57699999999999996,"15min":0.57899999999999996},"min":{"1min":0.38300000000000001,"5min":0.316,"15min":0.316},"max":{"1min":0.83699999999999997,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.60199999999999998,"15min":0.60499999999999998},"min":{"1min":0.38,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.59499999999999997}]},{"osd":6,"last update":"Sun Apr 7 14:37:07 2024","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.89000000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.60199999999999998}]},{"osd":7,"last update":"Sun Apr 7 14:37:32 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.57399999999999995}]}]},{"osd":0,"up_from":8,"seq":34359738422,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6928,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738224,"statfs":{"total":95995035648,"available":95987941376,"internally_reserved":0,"allocated":1122304,"data_stored":769588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Apr 7 14:37:14 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.33200000000000002,"15min":0.311},"min":{"1min":0.27600000000000002,"5min":0.218,"15min":0.218},"max":{"1min":0.51800000000000002,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.34200000000000003,"15min":0.317},"min":{"1min":0.25800000000000001,"5min":0.222,"15min":0.222},"max":{"1min":0.56999999999999995,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.25600000000000001}]},{"osd":2,"last update":"Sun Apr 7 14:37:40 2024","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.38300000000000001,"15min":0.376},"min":{"1min":0.28799999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.48999999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.31},{"interface":"front","average":{"1min":0.36899999999999999,"5min":0.373,"15min":0.36399999999999999},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":0.57399999999999995,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.33000000000000002}]},{"osd":3,"last update":"Sun Apr 7 14:37:03 2024","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.44900000000000001,"15min":0.45000000000000001},"min":{"1min":0.33800000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.55700000000000005,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.41299999999999998,"15min":0.41499999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.57199999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.28100000000000003}]},{"osd":4,"last update":"Sun Apr 7 14:37:25 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.56100000000000005,"15min":0.55800000000000005},"min":{"1min":0.40100000000000002,"5min":0.377,"15min":0.377},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.84599999999999997,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.45200000000000001}]},{"osd":5,"last update":"Sun Apr 7 14:37:45 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.56499999999999995,"15min":0.56699999999999995},"min":{"1min":0.38100000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.57699999999999996,"15min":0.57599999999999996},"min":{"1min":0.436,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.42099999999999999}]},{"osd":6,"last update":"Sun Apr 7 14:37:08 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.48899999999999999}]},{"osd":7,"last update":"Sun Apr 7 14:37:31 2024","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.44},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.51000000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411372,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6216,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738936,"statfs":{"total":95995035648,"available":95988670464,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Apr 7 14:37:39 2024","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.35299999999999998,"15min":0.34599999999999997},"min":{"1min":0.251,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.56999999999999995,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.34999999999999998,"5min":0.35899999999999999,"15min":0.34899999999999998},"min":{"1min":0.23699999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.66400000000000003,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.35699999999999998}]},{"osd":1,"last update":"Sun Apr 7 14:37:39 2024","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":0.36399999999999999,"15min":0.35799999999999998},"min":{"1min":0.254,"5min":0.215,"15min":0.215},"max":{"1min":0.61199999999999999,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.38700000000000001,"15min":0.379},"min":{"1min":0.27900000000000003,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.63200000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.33200000000000002}]},{"osd":3,"last update":"Sun Apr 7 14:37:02 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.40799999999999997,"15min":0.40600000000000003},"min":{"1min":0.28699999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.625,"5min":0.625,"15min":0.625},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.434,"5min":0.41799999999999998,"15min":0.41499999999999998},"min":{"1min":0.33700000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.29599999999999999}]},{"osd":4,"last update":"Sun Apr 7 14:37:27 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56999999999999995,"15min":0.57099999999999995},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.76500000000000001,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.624},{"interface":"front","average":{"1min":0.55100000000000005,"5min":1.272,"15min":1.3919999999999999},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.77600000000000002,"5min":18.847999999999999,"15min":18.847999999999999},"last":0.60399999999999998}]},{"osd":5,"last update":"Sun Apr 7 14:37:45 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.58899999999999997,"15min":0.59599999999999997},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.59299999999999997,"15min":0.59399999999999997},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.53200000000000003}]},{"osd":6,"last update":"Sun Apr 7 14:37:07 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.58699999999999997}]},{"osd":7,"last update":"Sun Apr 7 14:37:31 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.55100000000000005}]}]},{"osd":3,"up_from":26,"seq":111669149734,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5704,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739448,"statfs":{"total":95995035648,"available":95989194752,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Apr 7 14:37:05 2024","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.38200000000000001,"15min":0.38500000000000001},"min":{"1min":0.26000000000000001,"5min":0.254,"15min":0.254},"max":{"1min":0.57899999999999996,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.28599999999999998},{"interface":"front","average":{"1min":0.435,"5min":0.38600000000000001,"15min":0.378},"min":{"1min":0.307,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":1.044,"5min":1.044,"15min":1.044},"last":0.45600000000000002}]},{"osd":1,"last update":"Sun Apr 7 14:37:05 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.43099999999999999,"15min":0.42099999999999999},"min":{"1min":0.35199999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.41699999999999998,"15min":0.41899999999999998},"min":{"1min":0.29799999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.64300000000000002,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.35199999999999998}]},{"osd":2,"last update":"Sun Apr 7 14:37:05 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.32100000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.627,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.435,"5min":0.439,"15min":0.44},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.61699999999999999,"5min":1.0900000000000001,"15min":1.0900000000000001},"last":0.308}]},{"osd":4,"last update":"Sun Apr 7 14:37:24 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.76600000000000001,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.99099999999999999,"5min":0.99099999999999999,"15min":0.99099999999999999},"last":0.42399999999999999}]},{"osd":5,"last update":"Sun Apr 7 14:37:45 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":1.0189999999999999,"15min":1.0940000000000001},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.77600000000000002,"5min":10.353999999999999,"15min":10.353999999999999},"last":0.438},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":1.0469999999999999,"5min":1.0469999999999999,"15min":1.0469999999999999},"last":0.49199999999999999}]},{"osd":6,"last update":"Sun Apr 7 14:37:05 2024","interfaces":[{"interface":"back","average":{"1min":1.1830000000000001,"5min":1.1830000000000001,"15min":1.1830000000000001},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":10.388,"5min":10.388,"15min":10.388},"last":0.505},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.46500000000000002}]},{"osd":7,"last update":"Sun Apr 7 14:37:28 2024","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.1020000000000001,"5min":1.1020000000000001,"15min":1.1020000000000001},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.53000000000000003}]}]},{"osd":4,"up_from":31,"seq":133143986210,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5768,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739384,"statfs":{"total":95995035648,"available":95989129216,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Apr 7 14:37:22 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.48599999999999999,"15min":0.48099999999999998},"min":{"1min":0.40899999999999997,"5min":0.193,"15min":0.193},"max":{"1min":0.63300000000000001,"5min":0.64200000000000002,"15min":0.64200000000000002},"last":0.437},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.51100000000000001,"15min":0.51000000000000001},"min":{"1min":0.43099999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.67500000000000004,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.52700000000000002}]},{"osd":1,"last update":"Sun Apr 7 14:37:22 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.48399999999999999,"15min":0.47799999999999998},"min":{"1min":0.40999999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.63700000000000001,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.52000000000000002,"15min":0.51600000000000001},"min":{"1min":0.44400000000000001,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.68500000000000005,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.44800000000000001}]},{"osd":2,"last update":"Sun Apr 7 14:37:22 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.54800000000000004,"15min":0.54400000000000004},"min":{"1min":0.38700000000000001,"5min":0.372,"15min":0.372},"max":{"1min":0.70799999999999996,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.53300000000000003,"15min":0.53100000000000003},"min":{"1min":0.38100000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.57899999999999996}]},{"osd":3,"last update":"Sun Apr 7 14:37:22 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.496,"15min":0.48999999999999999},"min":{"1min":0.39400000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.66600000000000004,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.51000000000000001,"15min":0.504},"min":{"1min":0.42999999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.67300000000000004,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.56399999999999995}]},{"osd":5,"last update":"Sun Apr 7 14:37:48 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.372,"15min":0.36699999999999999},"min":{"1min":0.314,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.40400000000000003,"15min":0.39600000000000002},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.51500000000000001}]},{"osd":6,"last update":"Sun Apr 7 14:37:05 2024","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.59099999999999997}]},{"osd":7,"last update":"Sun Apr 7 14:37:31 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.60099999999999998}]}]},{"osd":5,"up_from":39,"seq":167503724574,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5700,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739452,"statfs":{"total":95995035648,"available":95989198848,"internally_reserved":0,"allocated":585728,"data_stored":240337,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Apr 7 14:37:46 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.52500000000000002,"15min":0.52600000000000002},"min":{"1min":0.36899999999999999,"5min":0.309,"15min":0.309},"max":{"1min":0.73499999999999999,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.52600000000000002,"15min":0.52400000000000002},"min":{"1min":0.36399999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.746,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.442}]},{"osd":1,"last update":"Sun Apr 7 14:37:46 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.80600000000000005,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.80600000000000005},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.54200000000000004,"15min":0.54000000000000004},"min":{"1min":0.45600000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.76900000000000002,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.72499999999999998}]},{"osd":2,"last update":"Sun Apr 7 14:37:46 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.51900000000000002,"15min":0.51700000000000002},"min":{"1min":0.44,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.79500000000000004,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.379,"5min":0.378,"15min":0.378},"max":{"1min":0.78400000000000003,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.63200000000000001}]},{"osd":3,"last update":"Sun Apr 7 14:37:46 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.55100000000000005,"15min":0.55500000000000005},"min":{"1min":0.34999999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.81299999999999994,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.53400000000000003}]},{"osd":4,"last update":"Sun Apr 7 14:37:46 2024","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.42699999999999999,"15min":0.41999999999999998},"min":{"1min":0.316,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.67600000000000005,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.503},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.432,"15min":0.42599999999999999},"min":{"1min":0.30399999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.51600000000000001}]},{"osd":6,"last update":"Sun Apr 7 14:37:08 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.69999999999999996}]},{"osd":7,"last update":"Sun Apr 7 14:37:32 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.54500000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-07T14:38:05.383 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph pg dump --format=json 2024-04-07T14:38:06.070 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-07T14:38:06.335 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:05 smithi083 ceph-mon[24810]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-07T14:38:06.335 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:05 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:06.335 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:05 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:06.335 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:05 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:06.335 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:05 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-07T14:38:06.335 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:05 smithi083 ceph-mon[24810]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-07T14:38:06.335 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:05 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:06.335 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:05 smithi083 ceph-mon[24810]: Deploying daemon grafana.a on smithi189 2024-04-07T14:38:06.335 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:05 smithi083 ceph-mon[29774]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-07T14:38:06.335 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:05 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:06.336 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:05 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:06.336 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:05 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:06.336 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:05 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-07T14:38:06.336 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:05 smithi083 ceph-mon[29774]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-07T14:38:06.336 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:05 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:06.336 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:05 smithi083 ceph-mon[29774]: Deploying daemon grafana.a on smithi189 2024-04-07T14:38:06.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:05 smithi189 ceph-mon[25797]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-07T14:38:06.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:05 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:06.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:05 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:06.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:05 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:06.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:05 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-07T14:38:06.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:05 smithi189 ceph-mon[25797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-07T14:38:06.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:05 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:06.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:05 smithi189 ceph-mon[25797]: Deploying daemon grafana.a on smithi189 2024-04-07T14:38:06.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:06 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[49483]: level=info ts=2024-04-07T14:38:06.818Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000852419s 2024-04-07T14:38:07.292 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:06 smithi083 ceph-mon[24810]: pgmap v49: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:38:07.293 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:06 smithi083 ceph-mon[29774]: pgmap v49: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:38:07.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:06 smithi189 ceph-mon[25797]: pgmap v49: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:38:08.708 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:38:08 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:38:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:38:09.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:08 smithi083 ceph-mon[24810]: pgmap v50: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:38:09.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:08 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:09.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:08 smithi083 ceph-mon[29774]: pgmap v50: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:38:09.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:08 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:09.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:08 smithi189 ceph-mon[25797]: pgmap v50: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:38:09.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:08 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:10.894 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-07T14:38:10.895 INFO:teuthology.orchestra.run.smithi083.stderr:dumped all 2024-04-07T14:38:11.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:10 smithi083 ceph-mon[24810]: pgmap v51: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:38:11.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:10 smithi083 ceph-mon[29774]: pgmap v51: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:38:11.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:10 smithi189 ceph-mon[25797]: pgmap v51: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:38:11.675 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:38:11 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:38:11] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:38:11.800 INFO:teuthology.orchestra.run.smithi083.stdout:{"pg_ready":true,"pg_map":{"version":51,"stamp":"2024-04-07T14:38:09.352308+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48900,"kb_used_data":6148,"kb_used_omap":0,"kb_used_meta":42688,"kb_avail":749912316,"statfs":{"total":767960285184,"available":767910211584,"internally_reserved":0,"allocated":6295552,"data_stored":3510644,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43712512},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002268"},"pg_stats":[{"pgid":"1.0","version":"60'128","reported_seq":266,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-04-07T14:36:49.000086+0000","last_change":"2024-04-07T14:36:32.319838+0000","last_active":"2024-04-07T14:36:49.000086+0000","last_peered":"2024-04-07T14:36:49.000086+0000","last_clean":"2024-04-07T14:36:49.000086+0000","last_became_active":"2024-04-07T14:36:32.319439+0000","last_became_peered":"2024-04-07T14:36:32.319439+0000","last_unstale":"2024-04-07T14:36:49.000086+0000","last_undegraded":"2024-04-07T14:36:49.000086+0000","last_fullsized":"2024-04-07T14:36:49.000086+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-07T14:34:36.446853+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-07T14:34:36.446853+0000","last_clean_scrub_stamp":"2024-04-07T14:34:36.446853+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-09T01:45:25.294322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201302,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6092,"kb_used_data":1092,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739060,"statfs":{"total":95995035648,"available":95988797440,"internally_reserved":0,"allocated":1118208,"data_stored":769323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Apr 7 14:37:28 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.64100000000000001}]},{"osd":1,"last update":"Sun Apr 7 14:37:28 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.437}]},{"osd":2,"last update":"Sun Apr 7 14:37:28 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.58399999999999996}]},{"osd":3,"last update":"Sun Apr 7 14:37:28 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.69899999999999995}]},{"osd":4,"last update":"Sun Apr 7 14:37:28 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.627},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.60999999999999999}]},{"osd":5,"last update":"Sun Apr 7 14:37:28 2024","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.73799999999999999},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.68300000000000005}]},{"osd":6,"last update":"Sun Apr 7 14:37:28 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.76500000000000001},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.71799999999999997}]}]},{"osd":6,"up_from":47,"seq":201863462938,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6152,"kb_used_data":1088,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739000,"statfs":{"total":95995035648,"available":95988736000,"internally_reserved":0,"allocated":1114112,"data_stored":769000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Apr 7 14:37:05 2024","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.47799999999999998}]},{"osd":1,"last update":"Sun Apr 7 14:37:05 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.64400000000000002}]},{"osd":2,"last update":"Sun Apr 7 14:37:05 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.58999999999999997}]},{"osd":3,"last update":"Sun Apr 7 14:37:05 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.72399999999999998},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.69099999999999995}]},{"osd":4,"last update":"Sun Apr 7 14:37:05 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.79000000000000004},{"interface":"front","average":{"1min":0.443,"5min":0.443,"15min":0.443},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.749}]},{"osd":5,"last update":"Sun Apr 7 14:37:05 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.76300000000000001},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.82799999999999996}]},{"osd":7,"last update":"Sun Apr 7 14:37:30 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.80300000000000005},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.86199999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574898,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6340,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738812,"statfs":{"total":95995035648,"available":95988543488,"internally_reserved":0,"allocated":585728,"data_stored":240416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Apr 7 14:37:12 2024","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.35099999999999998,"15min":0.34499999999999997},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.64100000000000001,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.316},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.36399999999999999,"15min":0.34799999999999998},"min":{"1min":0.26700000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.27400000000000002}]},{"osd":2,"last update":"Sun Apr 7 14:37:41 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.39500000000000002,"15min":0.39300000000000002},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.23200000000000001},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.44700000000000001,"15min":0.441},"min":{"1min":0.311,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.34499999999999997}]},{"osd":3,"last update":"Sun Apr 7 14:38:03 2024","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.42099999999999999,"15min":0.42899999999999999},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.376,"5min":0.42399999999999999,"15min":0.438},"min":{"1min":0.26300000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.253}]},{"osd":4,"last update":"Sun Apr 7 14:37:22 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.54700000000000004,"15min":0.54500000000000004},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.77100000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.55600000000000005,"15min":0.55000000000000004},"min":{"1min":0.38500000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.59499999999999997}]},{"osd":5,"last update":"Sun Apr 7 14:37:49 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.57699999999999996,"15min":0.57899999999999996},"min":{"1min":0.38300000000000001,"5min":0.316,"15min":0.316},"max":{"1min":0.83699999999999997,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.60199999999999998,"15min":0.60499999999999998},"min":{"1min":0.38,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.58299999999999996}]},{"osd":6,"last update":"Sun Apr 7 14:37:07 2024","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.89000000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.53800000000000003}]},{"osd":7,"last update":"Sun Apr 7 14:37:32 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.61899999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738423,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6928,"kb_used_data":1096,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738224,"statfs":{"total":95995035648,"available":95987941376,"internally_reserved":0,"allocated":1122304,"data_stored":769588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Apr 7 14:37:14 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.33200000000000002,"15min":0.311},"min":{"1min":0.27600000000000002,"5min":0.218,"15min":0.218},"max":{"1min":0.51800000000000002,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.34200000000000003,"15min":0.317},"min":{"1min":0.25800000000000001,"5min":0.222,"15min":0.222},"max":{"1min":0.56999999999999995,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.33000000000000002}]},{"osd":2,"last update":"Sun Apr 7 14:37:40 2024","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.38300000000000001,"15min":0.376},"min":{"1min":0.28799999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.48999999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.36899999999999999,"5min":0.373,"15min":0.36399999999999999},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":0.57399999999999995,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.35299999999999998}]},{"osd":3,"last update":"Sun Apr 7 14:38:04 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.438,"15min":0.44600000000000001},"min":{"1min":0.30399999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.58499999999999996,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.375,"5min":0.40500000000000003,"15min":0.41199999999999998},"min":{"1min":0.28100000000000003,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.623,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.48499999999999999}]},{"osd":4,"last update":"Sun Apr 7 14:37:25 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.56100000000000005,"15min":0.55800000000000005},"min":{"1min":0.40100000000000002,"5min":0.377,"15min":0.377},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.84599999999999997,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.56299999999999994}]},{"osd":5,"last update":"Sun Apr 7 14:37:45 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.56499999999999995,"15min":0.56699999999999995},"min":{"1min":0.38100000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.57699999999999996,"15min":0.57599999999999996},"min":{"1min":0.436,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.64500000000000002}]},{"osd":6,"last update":"Sun Apr 7 14:37:08 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.73399999999999999}]},{"osd":7,"last update":"Sun Apr 7 14:37:31 2024","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.71099999999999997}]}]},{"osd":2,"up_from":18,"seq":77309411373,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6216,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738936,"statfs":{"total":95995035648,"available":95988670464,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Apr 7 14:37:39 2024","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.35299999999999998,"15min":0.34599999999999997},"min":{"1min":0.251,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.56999999999999995,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.28000000000000003},{"interface":"front","average":{"1min":0.34999999999999998,"5min":0.35899999999999999,"15min":0.34899999999999998},"min":{"1min":0.23699999999999999,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.66400000000000003,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.33100000000000002}]},{"osd":1,"last update":"Sun Apr 7 14:37:39 2024","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":0.36399999999999999,"15min":0.35799999999999998},"min":{"1min":0.254,"5min":0.215,"15min":0.215},"max":{"1min":0.61199999999999999,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.38700000000000001,"15min":0.379},"min":{"1min":0.27900000000000003,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.63200000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.36799999999999999}]},{"osd":3,"last update":"Sun Apr 7 14:38:03 2024","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.40699999999999997,"15min":0.40500000000000003},"min":{"1min":0.30499999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.41299999999999998,"15min":0.41299999999999998},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.59799999999999998,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.39100000000000001}]},{"osd":4,"last update":"Sun Apr 7 14:37:27 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56999999999999995,"15min":0.57099999999999995},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.76500000000000001,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.55100000000000005,"5min":1.272,"15min":1.3919999999999999},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.77600000000000002,"5min":18.847999999999999,"15min":18.847999999999999},"last":0.70499999999999996}]},{"osd":5,"last update":"Sun Apr 7 14:37:45 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.58899999999999997,"15min":0.59599999999999997},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.59299999999999997,"15min":0.59399999999999997},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.755}]},{"osd":6,"last update":"Sun Apr 7 14:37:07 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.71399999999999997}]},{"osd":7,"last update":"Sun Apr 7 14:37:31 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.747}]}]},{"osd":3,"up_from":26,"seq":111669149735,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5704,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739448,"statfs":{"total":95995035648,"available":95989194752,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Apr 7 14:37:05 2024","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":0.38200000000000001,"15min":0.38500000000000001},"min":{"1min":0.26000000000000001,"5min":0.254,"15min":0.254},"max":{"1min":0.57899999999999996,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.23400000000000001},{"interface":"front","average":{"1min":0.435,"5min":0.38600000000000001,"15min":0.378},"min":{"1min":0.307,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":1.044,"5min":1.044,"15min":1.044},"last":0.252}]},{"osd":1,"last update":"Sun Apr 7 14:37:05 2024","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.43099999999999999,"15min":0.42099999999999999},"min":{"1min":0.35199999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.41699999999999998,"15min":0.41899999999999998},"min":{"1min":0.29799999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.64300000000000002,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.27900000000000003}]},{"osd":2,"last update":"Sun Apr 7 14:37:05 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.32100000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.627,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.308},{"interface":"front","average":{"1min":0.435,"5min":0.439,"15min":0.44},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.61699999999999999,"5min":1.0900000000000001,"15min":1.0900000000000001},"last":0.27000000000000002}]},{"osd":4,"last update":"Sun Apr 7 14:37:24 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.76600000000000001,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.99099999999999999,"5min":0.99099999999999999,"15min":0.99099999999999999},"last":0.59899999999999998}]},{"osd":5,"last update":"Sun Apr 7 14:37:45 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":1.0189999999999999,"15min":1.0940000000000001},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.77600000000000002,"5min":10.353999999999999,"15min":10.353999999999999},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":1.0469999999999999,"5min":1.0469999999999999,"15min":1.0469999999999999},"last":0.66100000000000003}]},{"osd":6,"last update":"Sun Apr 7 14:37:05 2024","interfaces":[{"interface":"back","average":{"1min":1.1830000000000001,"5min":1.1830000000000001,"15min":1.1830000000000001},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":10.388,"5min":10.388,"15min":10.388},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.58399999999999996}]},{"osd":7,"last update":"Sun Apr 7 14:37:28 2024","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.1020000000000001,"5min":1.1020000000000001,"15min":1.1020000000000001},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.66900000000000004}]}]},{"osd":4,"up_from":31,"seq":133143986211,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5768,"kb_used_data":576,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739384,"statfs":{"total":95995035648,"available":95989129216,"internally_reserved":0,"allocated":589824,"data_stored":240660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Apr 7 14:37:22 2024","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.48599999999999999,"15min":0.48099999999999998},"min":{"1min":0.40899999999999997,"5min":0.193,"15min":0.193},"max":{"1min":0.63300000000000001,"5min":0.64200000000000002,"15min":0.64200000000000002},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.51100000000000001,"15min":0.51000000000000001},"min":{"1min":0.43099999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.67500000000000004,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.38900000000000001}]},{"osd":1,"last update":"Sun Apr 7 14:37:22 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.48399999999999999,"15min":0.47799999999999998},"min":{"1min":0.40999999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.63700000000000001,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.52000000000000002,"15min":0.51600000000000001},"min":{"1min":0.44400000000000001,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.68500000000000005,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.48799999999999999}]},{"osd":2,"last update":"Sun Apr 7 14:37:22 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.54800000000000004,"15min":0.54400000000000004},"min":{"1min":0.38700000000000001,"5min":0.372,"15min":0.372},"max":{"1min":0.70799999999999996,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.53300000000000003,"15min":0.53100000000000003},"min":{"1min":0.38100000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.45100000000000001}]},{"osd":3,"last update":"Sun Apr 7 14:37:22 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.496,"15min":0.48999999999999999},"min":{"1min":0.39400000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.66600000000000004,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.51000000000000001,"15min":0.504},"min":{"1min":0.42999999999999999,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.67300000000000004,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.49399999999999999}]},{"osd":5,"last update":"Sun Apr 7 14:37:48 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.372,"15min":0.36699999999999999},"min":{"1min":0.314,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.45400000000000001,"5min":0.40400000000000003,"15min":0.39600000000000002},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.311}]},{"osd":6,"last update":"Sun Apr 7 14:37:05 2024","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.44}]},{"osd":7,"last update":"Sun Apr 7 14:37:31 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.42999999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724575,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5700,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739452,"statfs":{"total":95995035648,"available":95989198848,"internally_reserved":0,"allocated":585728,"data_stored":240337,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Apr 7 14:37:46 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.52500000000000002,"15min":0.52600000000000002},"min":{"1min":0.36899999999999999,"5min":0.309,"15min":0.309},"max":{"1min":0.73499999999999999,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.52600000000000002,"15min":0.52400000000000002},"min":{"1min":0.36399999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.746,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.48399999999999999}]},{"osd":1,"last update":"Sun Apr 7 14:37:46 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.80600000000000005,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.54200000000000004,"15min":0.54000000000000004},"min":{"1min":0.45600000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.76900000000000002,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.54700000000000004}]},{"osd":2,"last update":"Sun Apr 7 14:37:46 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.51900000000000002,"15min":0.51700000000000002},"min":{"1min":0.44,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.79500000000000004,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.379,"5min":0.378,"15min":0.378},"max":{"1min":0.78400000000000003,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.46400000000000002}]},{"osd":3,"last update":"Sun Apr 7 14:37:46 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.55100000000000005,"15min":0.55500000000000005},"min":{"1min":0.34999999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.81299999999999994,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.497},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.51600000000000001}]},{"osd":4,"last update":"Sun Apr 7 14:37:46 2024","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.42699999999999999,"15min":0.41999999999999998},"min":{"1min":0.316,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.67600000000000005,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.432,"15min":0.42599999999999999},"min":{"1min":0.30399999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.53300000000000003}]},{"osd":6,"last update":"Sun Apr 7 14:37:08 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.374},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.58899999999999997}]},{"osd":7,"last update":"Sun Apr 7 14:37:32 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.45100000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-07T14:38:11.801 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-07T14:38:11.801 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-07T14:38:11.801 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-07T14:38:11.802 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-07T14:38:11.982 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:38:11 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:38:11] "GET /metrics HTTP/1.1" 200 191576 "" "Prometheus/2.33.4" 2024-04-07T14:38:12.319 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:11 smithi083 ceph-mon[24810]: from='client.14640 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-07T14:38:12.319 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:11 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:12.319 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:11 smithi083 ceph-mon[29774]: from='client.14640 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-07T14:38:12.319 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:11 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:12.361 INFO:teuthology.orchestra.run.smithi083.stderr:Inferring config /var/lib/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-07T14:38:12.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:11 smithi189 ceph-mon[25797]: from='client.14640 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-07T14:38:12.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:11 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:12.588 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:38:12 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:38:12] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:38:13.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:12 smithi083 ceph-mon[24810]: pgmap v52: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:38:13.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:12 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:13.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:12 smithi083 ceph-mon[29774]: pgmap v52: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:38:13.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:12 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:13.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:12 smithi189 ceph-mon[25797]: pgmap v52: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:38:13.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:12 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:14.820 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:38:14 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:38:14] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:38:15.119 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:14 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[49483]: level=info ts=2024-04-07T14:38:14.822Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.004547446s 2024-04-07T14:38:15.120 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:14 smithi083 ceph-mon[24810]: pgmap v53: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:38:15.120 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:14 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:15.120 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:14 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:15.120 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:14 smithi083 ceph-mon[29774]: pgmap v53: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:38:15.120 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:14 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:15.120 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:14 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:15.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:14 smithi189 ceph-mon[25797]: pgmap v53: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:38:15.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:14 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:15.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:14 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:15.703 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-07T14:38:15.704 INFO:teuthology.orchestra.run.smithi083.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-07T14:38:16.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:16 smithi083 ceph-mon[24810]: pgmap v54: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:38:16.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:16 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/580871602' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-07T14:38:16.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:16 smithi083 ceph-mon[29774]: pgmap v54: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:38:16.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:16 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/580871602' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-07T14:38:16.522 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-07T14:38:16.523 INFO:tasks.cephadm:Setup complete, yielding 2024-04-07T14:38:16.523 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-07T14:38:16.537 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi083.front.sepia.ceph.com 2024-04-07T14:38:16.537 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -- bash -c 'ceph config set mgr mgr/cephadm/use_repo_digest false --force' 2024-04-07T14:38:16.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:16 smithi189 ceph-mon[25797]: pgmap v54: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:38:16.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:16 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/580871602' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-07T14:38:18.612 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:18 smithi083 ceph-mon[24810]: pgmap v55: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:38:18.612 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:18 smithi083 ceph-mon[29774]: pgmap v55: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:38:18.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:18 smithi189 ceph-mon[25797]: pgmap v55: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:38:18.708 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:38:18 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:38:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:38:19.318 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-07T14:38:19.329 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi083.front.sepia.ceph.com 2024-04-07T14:38:19.329 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -e sha1=e5c885fe95ee982ff7b45214daa61b1a6d09d57d -- bash -c 'radosgw-admin realm create --rgw-realm=r --default' 2024-04-07T14:38:19.771 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:38:19 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:38:19] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:38:19.772 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:19 smithi083 ceph-mon[24810]: from='client.? ' entity='client.admin' 2024-04-07T14:38:19.772 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:19 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:19.772 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:19 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:19.772 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:19 smithi083 ceph-mon[29774]: from='client.? ' entity='client.admin' 2024-04-07T14:38:19.772 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:19 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:19.772 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:19 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:19.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:19 smithi189 ceph-mon[25797]: from='client.? ' entity='client.admin' 2024-04-07T14:38:19.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:19 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:19.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:19 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:20.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:20 smithi083 ceph-mon[24810]: pgmap v56: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:38:20.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:20 smithi083 ceph-mon[29774]: pgmap v56: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:38:20.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:20 smithi189 ceph-mon[25797]: pgmap v56: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:38:21.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:38:21 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:38:21] "GET /metrics HTTP/1.1" 200 191578 "" "Prometheus/2.33.4" 2024-04-07T14:38:22.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:22 smithi189 ceph-mon[25797]: pgmap v57: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:38:22.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:22 smithi083 ceph-mon[24810]: pgmap v57: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:38:22.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:22 smithi083 ceph-mon[29774]: pgmap v57: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:38:23.654 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-04-07T14:38:23.654 INFO:teuthology.orchestra.run.smithi083.stdout: "id": "3a89d6cc-37c5-47ad-85a8-f8c0285f9dde", 2024-04-07T14:38:23.655 INFO:teuthology.orchestra.run.smithi083.stdout: "name": "r", 2024-04-07T14:38:23.655 INFO:teuthology.orchestra.run.smithi083.stdout: "current_period": "29108e7a-1a0f-4528-bc52-f38ccc4e2337", 2024-04-07T14:38:23.655 INFO:teuthology.orchestra.run.smithi083.stdout: "epoch": 1 2024-04-07T14:38:23.655 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-04-07T14:38:23.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:23 smithi083 ceph-mon[24810]: osdmap e61: 8 total, 8 up, 8 in 2024-04-07T14:38:23.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:23 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/3823698370' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-07T14:38:23.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:23 smithi083 ceph-mon[24810]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-07T14:38:23.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:23 smithi083 ceph-mon[29774]: osdmap e61: 8 total, 8 up, 8 in 2024-04-07T14:38:23.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:23 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/3823698370' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-07T14:38:23.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:23 smithi083 ceph-mon[29774]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-07T14:38:23.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:23 smithi189 ceph-mon[25797]: osdmap e61: 8 total, 8 up, 8 in 2024-04-07T14:38:23.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:23 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/3823698370' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-07T14:38:23.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:23 smithi189 ceph-mon[25797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-07T14:38:24.587 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 systemd[1]: Starting Ceph grafana.a for 7388281a-f4eb-11ee-b649-cb9ed24678a4... 2024-04-07T14:38:24.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:24 smithi083 ceph-mon[24810]: pgmap v59: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:38:24.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:24 smithi083 ceph-mon[24810]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-04-07T14:38:24.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:24 smithi083 ceph-mon[24810]: osdmap e62: 8 total, 8 up, 8 in 2024-04-07T14:38:24.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:24 smithi083 ceph-mon[29774]: pgmap v59: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:38:24.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:24 smithi083 ceph-mon[29774]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-04-07T14:38:24.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:24 smithi083 ceph-mon[29774]: osdmap e62: 8 total, 8 up, 8 in 2024-04-07T14:38:24.748 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -e sha1=e5c885fe95ee982ff7b45214daa61b1a6d09d57d -- bash -c 'radosgw-admin zonegroup create --rgw-zonegroup=default --master --default' 2024-04-07T14:38:24.840 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-mon[25797]: pgmap v59: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:38:24.840 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-mon[25797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-04-07T14:38:24.840 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-mon[25797]: osdmap e62: 8 total, 8 up, 8 in 2024-04-07T14:38:24.840 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 podman[44635]: 2024-04-07 14:38:24.491158256 +0000 UTC m=+0.017076878 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-04-07T14:38:24.840 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 podman[44635]: 2024-04-07 14:38:24.614417376 +0000 UTC m=+0.140336004 container create cea72126e7bd2b90832ad688c38a06eeadad601787b38a858219cc97177171ff (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a, maintainer=Paul Cuzner , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, io.openshift.expose-services=, name=ubi8, com.redhat.component=ubi8-container, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., build-date=2022-03-28T10:36:18.413762, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, vcs-type=git, release=236.1648460182, io.buildah.version=1.24.2, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.openshift.tags=base rhel8) 2024-04-07T14:38:24.840 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 podman[44635]: 2024-04-07 14:38:24.842168938 +0000 UTC m=+0.368087555 container init cea72126e7bd2b90832ad688c38a06eeadad601787b38a858219cc97177171ff (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-03-28T10:36:18.413762, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , io.openshift.expose-services=, distribution-scope=public, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, release=236.1648460182, vendor=Red Hat, Inc., io.buildah.version=1.24.2, architecture=x86_64, name=ubi8, com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container) 2024-04-07T14:38:25.090 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 podman[44635]: 2024-04-07 14:38:24.844870256 +0000 UTC m=+0.370788877 container start cea72126e7bd2b90832ad688c38a06eeadad601787b38a858219cc97177171ff (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a, build-date=2022-03-28T10:36:18.413762, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.24.2, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=236.1648460182, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, name=ubi8, io.openshift.tags=base rhel8, description=Ceph Grafana Container, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , version=8.5, vcs-type=git, distribution-scope=public, architecture=x86_64) 2024-04-07T14:38:25.091 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2024-04-07T14:38:25.091 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2024-04-07T14:38:25.091 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2024-04-07T14:38:25.091 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2024-04-07T14:38:25.091 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2024-04-07T14:38:25.091 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2024-04-07T14:38:25.091 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-04-07T14:38:25.091 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-04-07T14:38:25.091 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2024-04-07T14:38:25.091 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2024-04-07T14:38:25.091 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2024-04-07T14:38:25.091 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2024-04-07T14:38:25.092 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2024-04-07T14:38:25.092 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="App mode production" logger=settings 2024-04-07T14:38:25.092 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2024-04-07T14:38:25.092 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-04-07T14:38:25.092 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Starting DB migrations" logger=migrator 2024-04-07T14:38:25.092 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="create migration_log table" 2024-04-07T14:38:25.093 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="create user table" 2024-04-07T14:38:25.093 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.login" 2024-04-07T14:38:25.093 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user.email" 2024-04-07T14:38:25.093 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_login - v1" 2024-04-07T14:38:25.093 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_user_email - v1" 2024-04-07T14:38:25.093 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table user to user_v1 - v1" 2024-04-07T14:38:25.093 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="create user table v2" 2024-04-07T14:38:25.093 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_login - v2" 2024-04-07T14:38:25.093 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_user_email - v2" 2024-04-07T14:38:25.093 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2024-04-07T14:38:25.093 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table user_v1" 2024-04-07T14:38:25.093 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Add column help_flags1 to user table" 2024-04-07T14:38:25.093 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Update user table charset" 2024-04-07T14:38:25.093 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Add last_seen_at column to user" 2024-04-07T14:38:25.093 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Add missing user data" 2024-04-07T14:38:25.094 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_disabled column to user" 2024-04-07T14:38:25.094 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Add index user.login/user.email" 2024-04-07T14:38:25.094 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Add is_service_account column to user" 2024-04-07T14:38:25.094 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="create temp user table v1-7" 2024-04-07T14:38:25.094 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v1-7" 2024-04-07T14:38:25.094 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v1-7" 2024-04-07T14:38:25.094 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v1-7" 2024-04-07T14:38:25.094 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v1-7" 2024-04-07T14:38:25.094 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Update temp_user table charset" 2024-04-07T14:38:25.094 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_email - v1" 2024-04-07T14:38:25.094 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_org_id - v1" 2024-04-07T14:38:25.094 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_code - v1" 2024-04-07T14:38:25.094 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_temp_user_status - v1" 2024-04-07T14:38:25.094 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-04-07T14:38:25.095 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="create temp_user v2" 2024-04-07T14:38:25.095 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_email - v2" 2024-04-07T14:38:25.095 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_org_id - v2" 2024-04-07T14:38:25.095 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_code - v2" 2024-04-07T14:38:25.095 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_temp_user_status - v2" 2024-04-07T14:38:25.095 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="copy temp_user v1 to v2" 2024-04-07T14:38:25.095 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="drop temp_user_tmp_qwerty" 2024-04-07T14:38:25.095 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Set created for temp users that will otherwise prematurely expire" 2024-04-07T14:38:25.095 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="create star table" 2024-04-07T14:38:25.095 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index star.user_id_dashboard_id" 2024-04-07T14:38:25.095 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="create org table v1" 2024-04-07T14:38:25.095 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_name - v1" 2024-04-07T14:38:25.095 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="create org_user table v1" 2024-04-07T14:38:25.095 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_org_id - v1" 2024-04-07T14:38:25.095 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_org_user_org_id_user_id - v1" 2024-04-07T14:38:25.096 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_org_user_user_id - v1" 2024-04-07T14:38:25.096 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Update org table charset" 2024-04-07T14:38:25.096 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Update org_user table charset" 2024-04-07T14:38:25.096 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate all Read Only Viewers to Viewers" 2024-04-07T14:38:25.096 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard table" 2024-04-07T14:38:25.096 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard.account_id" 2024-04-07T14:38:25.096 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_account_id_slug" 2024-04-07T14:38:25.096 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_tag table" 2024-04-07T14:38:25.096 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_tag.dasboard_id_term" 2024-04-07T14:38:25.096 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-04-07T14:38:25.096 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard to dashboard_v1 - v1" 2024-04-07T14:38:25.096 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard v2" 2024-04-07T14:38:25.096 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_org_id - v2" 2024-04-07T14:38:25.096 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_org_id_slug - v2" 2024-04-07T14:38:25.096 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard v1 to v2" 2024-04-07T14:38:25.097 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_v1" 2024-04-07T14:38:25.097 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard.data to mediumtext v1" 2024-04-07T14:38:25.097 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Add column updated_by in dashboard - v2" 2024-04-07T14:38:25.097 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 bash[44635]: cea72126e7bd2b90832ad688c38a06eeadad601787b38a858219cc97177171ff 2024-04-07T14:38:25.097 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Add column created_by in dashboard - v2" 2024-04-07T14:38:25.097 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 systemd[1]: Started Ceph grafana.a for 7388281a-f4eb-11ee-b649-cb9ed24678a4. 2024-04-07T14:38:25.097 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Add column gnetId in dashboard" 2024-04-07T14:38:25.097 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for gnetId in dashboard" 2024-04-07T14:38:25.097 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_id in dashboard" 2024-04-07T14:38:25.097 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for plugin_id in dashboard" 2024-04-07T14:38:25.097 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_id in dashboard_tag" 2024-04-07T14:38:25.097 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard table charset" 2024-04-07T14:38:25.097 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_tag table charset" 2024-04-07T14:38:25.097 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Add column folder_id in dashboard" 2024-04-07T14:38:25.097 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Add column isFolder in dashboard" 2024-04-07T14:38:25.098 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Add column has_acl in dashboard" 2024-04-07T14:38:25.098 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in dashboard" 2024-04-07T14:38:25.099 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in dashboard" 2024-04-07T14:38:25.099 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index dashboard_org_id_uid" 2024-04-07T14:38:25.099 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_slug" 2024-04-07T14:38:25.099 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard title length" 2024-04-07T14:38:25.099 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index for dashboard_org_id_title_folder_id" 2024-04-07T14:38:25.099 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning" 2024-04-07T14:38:25.099 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-04-07T14:38:25.099 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_provisioning v2" 2024-04-07T14:38:25.099 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-04-07T14:38:25.099 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-04-07T14:38:25.099 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="copy dashboard_provisioning v1 to v2" 2024-04-07T14:38:25.099 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="drop dashboard_provisioning_tmp_qwerty" 2024-04-07T14:38:25.100 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Add check_sum column" 2024-04-07T14:38:25.100 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_title" 2024-04-07T14:38:25.100 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="delete tags for deleted dashboards" 2024-04-07T14:38:25.100 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="delete stars for deleted dashboards" 2024-04-07T14:38:25.100 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for dashboard_is_folder" 2024-04-07T14:38:25.100 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table" 2024-04-07T14:38:25.100 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="add index data_source.account_id" 2024-04-07T14:38:25.100 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index data_source.account_id_name" 2024-04-07T14:38:25.100 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_data_source_account_id - v1" 2024-04-07T14:38:25.100 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_data_source_account_id_name - v1" 2024-04-07T14:38:25.100 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table data_source to data_source_v1 - v1" 2024-04-07T14:38:25.100 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="create data_source table v2" 2024-04-07T14:38:25.100 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_data_source_org_id - v2" 2024-04-07T14:38:25.100 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_data_source_org_id_name - v2" 2024-04-07T14:38:25.100 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="copy data_source v1 to v2" 2024-04-07T14:38:25.100 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table data_source_v1 #2" 2024-04-07T14:38:25.101 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Add column with_credentials" 2024-04-07T14:38:25.101 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Add secure json data column" 2024-04-07T14:38:25.101 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Update data_source table charset" 2024-04-07T14:38:25.101 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Update initial version to 1" 2024-04-07T14:38:25.101 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Add read_only data column" 2024-04-07T14:38:25.101 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Migrate logging ds to loki ds" 2024-04-07T14:38:25.101 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Update json_data with nulls" 2024-04-07T14:38:25.101 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Add uid column" 2024-04-07T14:38:25.101 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid value" 2024-04-07T14:38:25.101 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index datasource_org_id_uid" 2024-04-07T14:38:25.101 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index datasource_org_id_is_default" 2024-04-07T14:38:25.101 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table" 2024-04-07T14:38:25.101 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id" 2024-04-07T14:38:25.101 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:24 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:24+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.key" 2024-04-07T14:38:25.101 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index api_key.account_id_name" 2024-04-07T14:38:25.102 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_api_key_account_id - v1" 2024-04-07T14:38:25.102 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_key - v1" 2024-04-07T14:38:25.102 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_api_key_account_id_name - v1" 2024-04-07T14:38:25.102 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table api_key to api_key_v1 - v1" 2024-04-07T14:38:25.102 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create api_key table v2" 2024-04-07T14:38:25.102 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_api_key_org_id - v2" 2024-04-07T14:38:25.102 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_key - v2" 2024-04-07T14:38:25.102 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_api_key_org_id_name - v2" 2024-04-07T14:38:25.102 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="copy api_key v1 to v2" 2024-04-07T14:38:25.102 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table api_key_v1" 2024-04-07T14:38:25.102 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Update api_key table charset" 2024-04-07T14:38:25.102 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add expires to api_key table" 2024-04-07T14:38:25.102 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add service account foreign key" 2024-04-07T14:38:25.102 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v4" 2024-04-07T14:38:25.102 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="drop table dashboard_snapshot_v4 #1" 2024-04-07T14:38:25.103 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_snapshot table v5 #2" 2024-04-07T14:38:25.103 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_key - v5" 2024-04-07T14:38:25.103 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-04-07T14:38:25.103 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_dashboard_snapshot_user_id - v5" 2024-04-07T14:38:25.103 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_snapshot to mediumtext v2" 2024-04-07T14:38:25.103 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Update dashboard_snapshot table charset" 2024-04-07T14:38:25.103 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external_delete_url to dashboard_snapshots table" 2024-04-07T14:38:25.103 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add encrypted dashboard json column" 2024-04-07T14:38:25.103 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-04-07T14:38:25.103 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create quota table v1" 2024-04-07T14:38:25.103 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_quota_org_id_user_id_target - v1" 2024-04-07T14:38:25.103 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Update quota table charset" 2024-04-07T14:38:25.103 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create plugin_setting table" 2024-04-07T14:38:25.103 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-04-07T14:38:25.103 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add column plugin_version to plugin_settings" 2024-04-07T14:38:25.104 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Update plugin_setting table charset" 2024-04-07T14:38:25.104 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create session table" 2024-04-07T14:38:25.104 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist table" 2024-04-07T14:38:25.104 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old table playlist_item table" 2024-04-07T14:38:25.104 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist table v2" 2024-04-07T14:38:25.104 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create playlist item table v2" 2024-04-07T14:38:25.104 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist table charset" 2024-04-07T14:38:25.104 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Update playlist_item table charset" 2024-04-07T14:38:25.104 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v2" 2024-04-07T14:38:25.104 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="drop preferences table v3" 2024-04-07T14:38:25.104 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create preferences table v3" 2024-04-07T14:38:25.104 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Update preferences table charset" 2024-04-07T14:38:25.104 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add column team_id in preferences" 2024-04-07T14:38:25.104 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Update team_id column values in preferences" 2024-04-07T14:38:25.105 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add column week_start in preferences" 2024-04-07T14:38:25.105 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create alert table v1" 2024-04-07T14:38:25.105 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert org_id & id " 2024-04-07T14:38:25.105 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert state" 2024-04-07T14:38:25.105 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert dashboard_id" 2024-04-07T14:38:25.105 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v1" 2024-04-07T14:38:25.105 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-04-07T14:38:25.105 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-04-07T14:38:25.105 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-04-07T14:38:25.105 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Create alert_rule_tag table v2" 2024-04-07T14:38:25.105 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-04-07T14:38:25.105 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="copy alert_rule_tag v1 to v2" 2024-04-07T14:38:25.105 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="drop table alert_rule_tag_v1" 2024-04-07T14:38:25.105 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification table v1" 2024-04-07T14:38:25.105 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add column is_default" 2024-04-07T14:38:25.106 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add column frequency" 2024-04-07T14:38:25.106 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add column send_reminder" 2024-04-07T14:38:25.106 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add column disable_resolve_message" 2024-04-07T14:38:25.106 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification org_id & name" 2024-04-07T14:38:25.106 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert table charset" 2024-04-07T14:38:25.106 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert_notification table charset" 2024-04-07T14:38:25.106 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create notification_journal table v1" 2024-04-07T14:38:25.106 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index notification_journal org_id & alert_id & notifier_id" 2024-04-07T14:38:25.106 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_notification_journal" 2024-04-07T14:38:25.106 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_notification_state table v1" 2024-04-07T14:38:25.106 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-04-07T14:38:25.106 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add for to alert table" 2024-04-07T14:38:25.106 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add column uid in alert_notification" 2024-04-07T14:38:25.107 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Update uid column values in alert_notification" 2024-04-07T14:38:25.107 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index alert_notification_org_id_uid" 2024-04-07T14:38:25.107 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index org_id_name" 2024-04-07T14:38:25.107 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add column secure_settings in alert_notification" 2024-04-07T14:38:25.107 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert.settings to mediumtext" 2024-04-07T14:38:25.107 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_notification_state_alert_id" 2024-04-07T14:38:25.107 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add non-unique index alert_rule_tag_alert_id" 2024-04-07T14:38:25.107 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Drop old annotation table v4" 2024-04-07T14:38:25.107 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create annotation table v5" 2024-04-07T14:38:25.107 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 0 v3" 2024-04-07T14:38:25.107 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 1 v3" 2024-04-07T14:38:25.107 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 2 v3" 2024-04-07T14:38:25.107 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 3 v3" 2024-04-07T14:38:25.107 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index annotation 4 v3" 2024-04-07T14:38:25.107 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Update annotation table charset" 2024-04-07T14:38:25.108 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add column region_id to annotation table" 2024-04-07T14:38:25.108 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Drop category_id index" 2024-04-07T14:38:25.108 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add column tags to annotation table" 2024-04-07T14:38:25.108 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v2" 2024-04-07T14:38:25.108 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add unique index annotation_tag.annotation_id_tag_id" 2024-04-07T14:38:25.286 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:38:24 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:38:24] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:38:25.355 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-04-07T14:38:25.356 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-04-07T14:38:25.356 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Create annotation_tag table v3" 2024-04-07T14:38:25.356 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-04-07T14:38:25.356 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="copy annotation_tag v2 to v3" 2024-04-07T14:38:25.356 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="drop table annotation_tag_v2" 2024-04-07T14:38:25.356 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Update alert annotations and set TEXT to empty" 2024-04-07T14:38:25.356 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add created time to annotation table" 2024-04-07T14:38:25.356 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add updated time to annotation table" 2024-04-07T14:38:25.356 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for created in annotation table" 2024-04-07T14:38:25.356 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for updated in annotation table" 2024-04-07T14:38:25.356 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Convert existing annotations from seconds to milliseconds" 2024-04-07T14:38:25.356 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add epoch_end column" 2024-04-07T14:38:25.357 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for epoch_end" 2024-04-07T14:38:25.357 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Make epoch_end the same as epoch" 2024-04-07T14:38:25.357 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Move region to single row" 2024-04-07T14:38:25.357 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch from annotation table" 2024-04-07T14:38:25.357 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-04-07T14:38:25.357 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-04-07T14:38:25.357 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for org_id_epoch_end_epoch on annotation table" 2024-04-07T14:38:25.357 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Remove index org_id_epoch_epoch_end from annotation table" 2024-04-07T14:38:25.357 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add index for alert_id on annotation table" 2024-04-07T14:38:25.357 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create test_data table" 2024-04-07T14:38:25.357 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard_version table v1" 2024-04-07T14:38:25.357 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_version.dashboard_id" 2024-04-07T14:38:25.358 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-04-07T14:38:25.358 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Set dashboard version to 1 where 0" 2024-04-07T14:38:25.358 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="save existing dashboard data in dashboard_version table v1" 2024-04-07T14:38:25.358 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="alter dashboard_version.data to mediumtext v1" 2024-04-07T14:38:25.358 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create team table" 2024-04-07T14:38:25.358 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index team.org_id" 2024-04-07T14:38:25.358 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_org_id_name" 2024-04-07T14:38:25.358 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create team member table" 2024-04-07T14:38:25.358 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.org_id" 2024-04-07T14:38:25.358 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_member_org_id_team_id_user_id" 2024-04-07T14:38:25.358 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_member.team_id" 2024-04-07T14:38:25.358 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add column email to team table" 2024-04-07T14:38:25.358 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add column external to team_member table" 2024-04-07T14:38:25.359 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add column permission to team_member table" 2024-04-07T14:38:25.359 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create dashboard acl table" 2024-04-07T14:38:25.360 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_dashboard_id" 2024-04-07T14:38:25.360 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_user_id" 2024-04-07T14:38:25.360 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index dashboard_acl_dashboard_id_team_id" 2024-04-07T14:38:25.360 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_user_id" 2024-04-07T14:38:25.360 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_team_id" 2024-04-07T14:38:25.360 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_acl_org_id_role" 2024-04-07T14:38:25.360 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index dashboard_permission" 2024-04-07T14:38:25.360 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="save default acl rules in dashboard_acl table" 2024-04-07T14:38:25.360 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="delete acl rules for deleted dashboards and folders" 2024-04-07T14:38:25.360 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create tag table" 2024-04-07T14:38:25.360 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index tag.key_value" 2024-04-07T14:38:25.360 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create login attempt table" 2024-04-07T14:38:25.360 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index login_attempt.username" 2024-04-07T14:38:25.360 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="drop index IDX_login_attempt_username - v1" 2024-04-07T14:38:25.360 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-04-07T14:38:25.361 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create login_attempt v2" 2024-04-07T14:38:25.361 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2024-04-07T14:38:25.361 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2024-04-07T14:38:25.361 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2024-04-07T14:38:25.361 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2024-04-07T14:38:25.361 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-04-07T14:38:25.361 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2024-04-07T14:38:25.361 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2024-04-07T14:38:25.361 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2024-04-07T14:38:25.361 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2024-04-07T14:38:25.361 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2024-04-07T14:38:25.361 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2024-04-07T14:38:25.361 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2024-04-07T14:38:25.361 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2024-04-07T14:38:25.362 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2024-04-07T14:38:25.362 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2024-04-07T14:38:25.362 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2024-04-07T14:38:25.362 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_auth_token.user_id" 2024-04-07T14:38:25.362 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add revoked_at to the user auth token" 2024-04-07T14:38:25.362 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2024-04-07T14:38:25.362 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2024-04-07T14:38:25.362 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create short_url table v1" 2024-04-07T14:38:25.362 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index short_url.org_id-uid" 2024-04-07T14:38:25.362 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition table" 2024-04-07T14:38:25.362 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition table" 2024-04-07T14:38:25.362 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and title columns" 2024-04-07T14:38:25.362 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and uid columns" 2024-04-07T14:38:25.362 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition table data column to mediumtext in mysql" 2024-04-07T14:38:25.362 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and title columns" 2024-04-07T14:38:25.362 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and uid columns" 2024-04-07T14:38:25.363 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and title columns" 2024-04-07T14:38:25.363 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and uid columns" 2024-04-07T14:38:25.363 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add column paused in alert_definition" 2024-04-07T14:38:25.363 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition table" 2024-04-07T14:38:25.363 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition_version table" 2024-04-07T14:38:25.363 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition_version table" 2024-04-07T14:38:25.363 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-04-07T14:38:25.363 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-04-07T14:38:25.363 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition_version table data column to mediumtext in mysql" 2024-04-07T14:38:25.363 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition_version table" 2024-04-07T14:38:25.363 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_instance table" 2024-04-07T14:38:25.363 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-04-07T14:38:25.363 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, current_state columns" 2024-04-07T14:38:25.363 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add column current_state_end to alert_instance" 2024-04-07T14:38:25.363 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-04-07T14:38:25.363 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, current_state on alert_instance" 2024-04-07T14:38:25.364 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_org_id to rule_org_id in alert_instance" 2024-04-07T14:38:25.364 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_uid to rule_uid in alert_instance" 2024-04-07T14:38:25.364 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-04-07T14:38:25.364 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, current_state on alert_instance" 2024-04-07T14:38:25.364 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule table" 2024-04-07T14:38:25.364 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and title columns" 2024-04-07T14:38:25.364 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and uid columns" 2024-04-07T14:38:25.364 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-04-07T14:38:25.364 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule table data column to mediumtext in mysql" 2024-04-07T14:38:25.364 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule" 2024-04-07T14:38:25.364 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule" 2024-04-07T14:38:25.364 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule" 2024-04-07T14:38:25.364 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="remove unique index from alert_rule on org_id, title columns" 2024-04-07T14:38:25.364 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-04-07T14:38:25.364 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add dashboard_uid column to alert_rule" 2024-04-07T14:38:25.364 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add panel_id column to alert_rule" 2024-04-07T14:38:25.364 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-04-07T14:38:25.365 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule_version table" 2024-04-07T14:38:25.365 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-04-07T14:38:25.365 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-04-07T14:38:25.365 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule_version table data column to mediumtext in mysql" 2024-04-07T14:38:25.365 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule_version" 2024-04-07T14:38:25.365 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule_version" 2024-04-07T14:38:25.365 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule_version" 2024-04-07T14:38:25.365 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id=create_alert_configuration_table 2024-04-07T14:38:25.365 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add column default in alert_configuration" 2024-04-07T14:38:25.365 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-04-07T14:38:25.365 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add column org_id in alert_configuration" 2024-04-07T14:38:25.365 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_configuration table on org_id column" 2024-04-07T14:38:25.365 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id=create_ngalert_configuration_table 2024-04-07T14:38:25.365 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index in ngalert_configuration on org_id column" 2024-04-07T14:38:25.365 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="clear migration entry \"remove unified alerting data\"" 2024-04-07T14:38:25.365 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="move dashboard alerts to unified alerting" 2024-04-07T14:38:25.366 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element table v1" 2024-04-07T14:38:25.366 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element org_id-folder_id-name-kind" 2024-04-07T14:38:25.366 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element_connection table v1" 2024-04-07T14:38:25.366 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element_connection element_id-kind-connection_id" 2024-04-07T14:38:25.366 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index library_element org_id_uid" 2024-04-07T14:38:25.366 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="clone move dashboard alerts to unified alerting" 2024-04-07T14:38:25.366 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create data_keys table" 2024-04-07T14:38:25.366 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create kv_store table v1" 2024-04-07T14:38:25.366 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index kv_store.org_id-namespace-key" 2024-04-07T14:38:25.366 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="update dashboard_uid and panel_id from existing annotations" 2024-04-07T14:38:25.366 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create permission table" 2024-04-07T14:38:25.366 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index permission.role_id" 2024-04-07T14:38:25.366 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_id_action_scope" 2024-04-07T14:38:25.366 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create role table" 2024-04-07T14:38:25.367 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add column display_name" 2024-04-07T14:38:25.367 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add column group_name" 2024-04-07T14:38:25.367 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index role.org_id" 2024-04-07T14:38:25.367 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_org_id_name" 2024-04-07T14:38:25.367 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index role_org_id_uid" 2024-04-07T14:38:25.367 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create team role table" 2024-04-07T14:38:25.367 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.org_id" 2024-04-07T14:38:25.367 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_role_org_id_team_id_role_id" 2024-04-07T14:38:25.367 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.team_id" 2024-04-07T14:38:25.367 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create user role table" 2024-04-07T14:38:25.367 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.org_id" 2024-04-07T14:38:25.367 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_role_org_id_user_id_role_id" 2024-04-07T14:38:25.367 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.user_id" 2024-04-07T14:38:25.367 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create builtin role table" 2024-04-07T14:38:25.367 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.role_id" 2024-04-07T14:38:25.367 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.name" 2024-04-07T14:38:25.368 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Add column org_id to builtin_role table" 2024-04-07T14:38:25.368 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.org_id" 2024-04-07T14:38:25.368 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_org_id_role_id_role" 2024-04-07T14:38:25.368 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index role_org_id_uid" 2024-04-07T14:38:25.368 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role.uid" 2024-04-07T14:38:25.368 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="create seed assignment table" 2024-04-07T14:38:25.368 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_role_name" 2024-04-07T14:38:25.368 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="migrations completed" logger=migrator performed=381 skipped=0 duration=458.657534ms 2024-04-07T14:38:25.368 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Created default organization" logger=sqlstore 2024-04-07T14:38:25.368 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2024-04-07T14:38:25.368 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2024-04-07T14:38:25.623 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2024-04-07T14:38:25.623 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2024-04-07T14:38:25.623 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2024-04-07T14:38:25.623 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-04-07T14:38:25.623 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2024-04-07T14:38:25.623 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="warming cache for startup" logger=ngalert 2024-04-07T14:38:25.623 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-04-07T14:38:25.623 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:38:25+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2024-04-07T14:38:25.853 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:25 smithi083 ceph-mon[24810]: osdmap e63: 8 total, 8 up, 8 in 2024-04-07T14:38:25.853 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:25 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:25.853 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:25 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:25.853 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:25 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:38:25.854 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:25 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:38:25.854 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:25 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:25.854 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:25 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:25.854 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:25 smithi083 ceph-mon[24810]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:38:25.854 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:25 smithi083 ceph-mon[24810]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-07T14:38:25.854 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:25 smithi083 ceph-mon[24810]: Reconfiguring daemon alertmanager.a on smithi083 2024-04-07T14:38:25.854 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:25 smithi083 ceph-mon[29774]: osdmap e63: 8 total, 8 up, 8 in 2024-04-07T14:38:25.854 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:25 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:25.854 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:25 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:25.854 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:25 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:38:25.854 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:25 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:38:25.855 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:25 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:25.855 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:25 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:25.855 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:25 smithi083 ceph-mon[29774]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:38:25.855 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:25 smithi083 ceph-mon[29774]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-07T14:38:25.855 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:25 smithi083 ceph-mon[29774]: Reconfiguring daemon alertmanager.a on smithi083 2024-04-07T14:38:25.884 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-mon[25797]: osdmap e63: 8 total, 8 up, 8 in 2024-04-07T14:38:25.884 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:25.884 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:25.884 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:38:25.885 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:38:25.885 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:25.885 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:25.885 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-mon[25797]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:38:25.885 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-mon[25797]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-07T14:38:25.885 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:25 smithi189 ceph-mon[25797]: Reconfiguring daemon alertmanager.a on smithi083 2024-04-07T14:38:26.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:26 smithi083 ceph-mon[24810]: pgmap v62: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-04-07T14:38:26.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:26 smithi083 ceph-mon[29774]: pgmap v62: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-04-07T14:38:26.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:26 smithi189 ceph-mon[25797]: pgmap v62: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-04-07T14:38:28.453 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:38:28 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:38:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:38:28.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:28 smithi189 ceph-mon[25797]: pgmap v63: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-04-07T14:38:28.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:28 smithi083 ceph-mon[24810]: pgmap v63: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-04-07T14:38:28.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:28 smithi083 ceph-mon[29774]: pgmap v63: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-04-07T14:38:29.588 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:29 smithi083 systemd[1]: Stopping Ceph alertmanager.a for 7388281a-f4eb-11ee-b649-cb9ed24678a4... 2024-04-07T14:38:29.713 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-04-07T14:38:29.713 INFO:teuthology.orchestra.run.smithi083.stdout: "id": "c91d3c32-b880-41b0-9802-f2466d027d40", 2024-04-07T14:38:29.713 INFO:teuthology.orchestra.run.smithi083.stdout: "name": "default", 2024-04-07T14:38:29.713 INFO:teuthology.orchestra.run.smithi083.stdout: "api_name": "default", 2024-04-07T14:38:29.713 INFO:teuthology.orchestra.run.smithi083.stdout: "is_master": "true", 2024-04-07T14:38:29.713 INFO:teuthology.orchestra.run.smithi083.stdout: "endpoints": [], 2024-04-07T14:38:29.714 INFO:teuthology.orchestra.run.smithi083.stdout: "hostnames": [], 2024-04-07T14:38:29.714 INFO:teuthology.orchestra.run.smithi083.stdout: "hostnames_s3website": [], 2024-04-07T14:38:29.714 INFO:teuthology.orchestra.run.smithi083.stdout: "master_zone": "", 2024-04-07T14:38:29.714 INFO:teuthology.orchestra.run.smithi083.stdout: "zones": [], 2024-04-07T14:38:29.714 INFO:teuthology.orchestra.run.smithi083.stdout: "placement_targets": [], 2024-04-07T14:38:29.714 INFO:teuthology.orchestra.run.smithi083.stdout: "default_placement": "", 2024-04-07T14:38:29.714 INFO:teuthology.orchestra.run.smithi083.stdout: "realm_id": "3a89d6cc-37c5-47ad-85a8-f8c0285f9dde", 2024-04-07T14:38:29.714 INFO:teuthology.orchestra.run.smithi083.stdout: "sync_policy": { 2024-04-07T14:38:29.714 INFO:teuthology.orchestra.run.smithi083.stdout: "groups": [] 2024-04-07T14:38:29.714 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-04-07T14:38:29.714 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-04-07T14:38:30.192 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:29 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:30.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:29 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:30.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:29 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:30.571 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -e sha1=e5c885fe95ee982ff7b45214daa61b1a6d09d57d -- bash -c 'radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default' 2024-04-07T14:38:30.883 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:30 smithi083 bash[51220]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager.a" found: no such container 2024-04-07T14:38:30.883 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:30 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[49483]: level=info ts=2024-04-07T14:38:30.748Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-04-07T14:38:30.883 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:30 smithi083 podman[51329]: 2024-04-07 14:38:30.768543368 +0000 UTC m=+0.132005294 container died f5a5d0d8ddf604e53529accc3e3d3ea56e6fd1eb5ab4c73c83c0dbbc5347bae3 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-07T14:38:31.056 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:30 smithi189 ceph-mon[25797]: pgmap v64: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 1 op/s 2024-04-07T14:38:31.057 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:30 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:31.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:30 smithi083 ceph-mon[29774]: pgmap v64: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 1 op/s 2024-04-07T14:38:31.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:30 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:31.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:30 smithi083 ceph-mon[24810]: pgmap v64: 33 pgs: 33 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 1 op/s 2024-04-07T14:38:31.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:30 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:31.177 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:38:30 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:38:30] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:38:31.490 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:31 smithi083 podman[51329]: 2024-04-07 14:38:31.217626967 +0000 UTC m=+0.581088890 container cleanup f5a5d0d8ddf604e53529accc3e3d3ea56e6fd1eb5ab4c73c83c0dbbc5347bae3 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-07T14:38:31.490 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:31 smithi083 bash[51329]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a 2024-04-07T14:38:31.490 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:31 smithi083 podman[51357]: 2024-04-07 14:38:31.416189143 +0000 UTC m=+0.647545938 container remove f5a5d0d8ddf604e53529accc3e3d3ea56e6fd1eb5ab4c73c83c0dbbc5347bae3 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-07T14:38:31.490 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:31 smithi083 podman[51357]: 2024-04-07 14:38:31.492089569 +0000 UTC m=+0.723446363 volume remove c681fc92f14be8d8b25c620467cbeca623f184bdc836711ed878fe2c11886802 2024-04-07T14:38:31.826 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:38:31 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:38:31] "GET /metrics HTTP/1.1" 200 191578 "" "Prometheus/2.33.4" 2024-04-07T14:38:32.676 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:38:32 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:38:32] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:38:33.051 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:32 smithi083 ceph-mon[24810]: pgmap v65: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 1.4 KiB/s wr, 2 op/s 2024-04-07T14:38:33.051 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:32 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:33.051 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:32 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:33.051 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:32 smithi083 ceph-mon[29774]: pgmap v65: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 1.4 KiB/s wr, 2 op/s 2024-04-07T14:38:33.052 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:32 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:33.052 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:32 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:33.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:32 smithi189 ceph-mon[25797]: pgmap v65: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 1.4 KiB/s wr, 2 op/s 2024-04-07T14:38:33.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:32 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:33.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:32 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:33.676 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:33 smithi083 bash[51396]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager.a" found: no such container 2024-04-07T14:38:33.676 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:33 smithi083 systemd[1]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@alertmanager.a.service: Deactivated successfully. 2024-04-07T14:38:33.676 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:33 smithi083 systemd[1]: Stopped Ceph alertmanager.a for 7388281a-f4eb-11ee-b649-cb9ed24678a4. 2024-04-07T14:38:33.676 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:33 smithi083 systemd[1]: Starting Ceph alertmanager.a for 7388281a-f4eb-11ee-b649-cb9ed24678a4... 2024-04-07T14:38:34.317 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-04-07T14:38:34.317 INFO:teuthology.orchestra.run.smithi083.stdout: "id": "f7959d84-9e92-4aa7-95a7-1c200af9e17f", 2024-04-07T14:38:34.317 INFO:teuthology.orchestra.run.smithi083.stdout: "name": "z", 2024-04-07T14:38:34.317 INFO:teuthology.orchestra.run.smithi083.stdout: "domain_root": "z.rgw.meta:root", 2024-04-07T14:38:34.317 INFO:teuthology.orchestra.run.smithi083.stdout: "control_pool": "z.rgw.control", 2024-04-07T14:38:34.317 INFO:teuthology.orchestra.run.smithi083.stdout: "gc_pool": "z.rgw.log:gc", 2024-04-07T14:38:34.317 INFO:teuthology.orchestra.run.smithi083.stdout: "lc_pool": "z.rgw.log:lc", 2024-04-07T14:38:34.317 INFO:teuthology.orchestra.run.smithi083.stdout: "log_pool": "z.rgw.log", 2024-04-07T14:38:34.317 INFO:teuthology.orchestra.run.smithi083.stdout: "intent_log_pool": "z.rgw.log:intent", 2024-04-07T14:38:34.317 INFO:teuthology.orchestra.run.smithi083.stdout: "usage_log_pool": "z.rgw.log:usage", 2024-04-07T14:38:34.318 INFO:teuthology.orchestra.run.smithi083.stdout: "roles_pool": "z.rgw.meta:roles", 2024-04-07T14:38:34.318 INFO:teuthology.orchestra.run.smithi083.stdout: "reshard_pool": "z.rgw.log:reshard", 2024-04-07T14:38:34.318 INFO:teuthology.orchestra.run.smithi083.stdout: "user_keys_pool": "z.rgw.meta:users.keys", 2024-04-07T14:38:34.318 INFO:teuthology.orchestra.run.smithi083.stdout: "user_email_pool": "z.rgw.meta:users.email", 2024-04-07T14:38:34.318 INFO:teuthology.orchestra.run.smithi083.stdout: "user_swift_pool": "z.rgw.meta:users.swift", 2024-04-07T14:38:34.318 INFO:teuthology.orchestra.run.smithi083.stdout: "user_uid_pool": "z.rgw.meta:users.uid", 2024-04-07T14:38:34.318 INFO:teuthology.orchestra.run.smithi083.stdout: "otp_pool": "z.rgw.otp", 2024-04-07T14:38:34.318 INFO:teuthology.orchestra.run.smithi083.stdout: "system_key": { 2024-04-07T14:38:34.318 INFO:teuthology.orchestra.run.smithi083.stdout: "access_key": "", 2024-04-07T14:38:34.318 INFO:teuthology.orchestra.run.smithi083.stdout: "secret_key": "" 2024-04-07T14:38:34.318 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-04-07T14:38:34.318 INFO:teuthology.orchestra.run.smithi083.stdout: "placement_pools": [ 2024-04-07T14:38:34.318 INFO:teuthology.orchestra.run.smithi083.stdout: { 2024-04-07T14:38:34.318 INFO:teuthology.orchestra.run.smithi083.stdout: "key": "default-placement", 2024-04-07T14:38:34.319 INFO:teuthology.orchestra.run.smithi083.stdout: "val": { 2024-04-07T14:38:34.319 INFO:teuthology.orchestra.run.smithi083.stdout: "index_pool": "z.rgw.buckets.index", 2024-04-07T14:38:34.319 INFO:teuthology.orchestra.run.smithi083.stdout: "storage_classes": { 2024-04-07T14:38:34.319 INFO:teuthology.orchestra.run.smithi083.stdout: "STANDARD": { 2024-04-07T14:38:34.319 INFO:teuthology.orchestra.run.smithi083.stdout: "data_pool": "z.rgw.buckets.data" 2024-04-07T14:38:34.319 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-04-07T14:38:34.319 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-04-07T14:38:34.319 INFO:teuthology.orchestra.run.smithi083.stdout: "data_extra_pool": "z.rgw.buckets.non-ec", 2024-04-07T14:38:34.319 INFO:teuthology.orchestra.run.smithi083.stdout: "index_type": 0 2024-04-07T14:38:34.319 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-04-07T14:38:34.319 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-04-07T14:38:34.319 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-04-07T14:38:34.319 INFO:teuthology.orchestra.run.smithi083.stdout: "realm_id": "3a89d6cc-37c5-47ad-85a8-f8c0285f9dde", 2024-04-07T14:38:34.319 INFO:teuthology.orchestra.run.smithi083.stdout: "notif_pool": "z.rgw.log:notif" 2024-04-07T14:38:34.319 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-04-07T14:38:34.997 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -e sha1=e5c885fe95ee982ff7b45214daa61b1a6d09d57d -- bash -c 'radosgw-admin period update --rgw-realm=r --commit' 2024-04-07T14:38:35.271 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:34 smithi083 ceph-mon[24810]: pgmap v66: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 420 B/s rd, 1.1 KiB/s wr, 1 op/s 2024-04-07T14:38:35.271 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:34 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:35.272 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:34 smithi083 ceph-mon[29774]: pgmap v66: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 420 B/s rd, 1.1 KiB/s wr, 1 op/s 2024-04-07T14:38:35.272 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:34 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:35.272 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:35 smithi083 podman[51628]: 2024-04-07 14:38:35.096940965 +0000 UTC m=+0.018276308 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-04-07T14:38:35.272 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:35 smithi083 podman[51628]: 2024-04-07 14:38:35.207979379 +0000 UTC m=+0.129314694 volume create 3a3e8a1591b73c5e964af041dd6baaf7e540f333da740d55ef0b104d4afb0296 2024-04-07T14:38:35.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:34 smithi189 ceph-mon[25797]: pgmap v66: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 420 B/s rd, 1.1 KiB/s wr, 1 op/s 2024-04-07T14:38:35.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:34 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:35.557 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:35 smithi083 podman[51628]: 2024-04-07 14:38:35.334022698 +0000 UTC m=+0.255358013 container create 9f4fc57a8aba2c3ec0d31b6a658b17fc81706051a1eac5d62baf9cad2b90d2f5 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-07T14:38:35.952 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:35 smithi083 podman[51628]: 2024-04-07 14:38:35.678611032 +0000 UTC m=+0.599946356 container init 9f4fc57a8aba2c3ec0d31b6a658b17fc81706051a1eac5d62baf9cad2b90d2f5 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-07T14:38:35.952 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:35 smithi083 podman[51628]: 2024-04-07 14:38:35.681371315 +0000 UTC m=+0.602706633 container start 9f4fc57a8aba2c3ec0d31b6a658b17fc81706051a1eac5d62baf9cad2b90d2f5 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-07T14:38:35.952 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:35 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=info ts=2024-04-07T14:38:35.691Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-04-07T14:38:35.952 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:35 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=info ts=2024-04-07T14:38:35.691Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-04-07T14:38:35.952 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:35 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=info ts=2024-04-07T14:38:35.692Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.83 port=9094 2024-04-07T14:38:35.952 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:35 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=info ts=2024-04-07T14:38:35.694Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-07T14:38:35.952 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:35 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=info ts=2024-04-07T14:38:35.724Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-07T14:38:35.952 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:35 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=info ts=2024-04-07T14:38:35.725Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-07T14:38:35.952 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:35 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=info ts=2024-04-07T14:38:35.727Z caller=main.go:518 msg=Listening address=:9093 2024-04-07T14:38:35.952 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:35 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=info ts=2024-04-07T14:38:35.727Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-04-07T14:38:35.952 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:35 smithi083 bash[51628]: 9f4fc57a8aba2c3ec0d31b6a658b17fc81706051a1eac5d62baf9cad2b90d2f5 2024-04-07T14:38:35.953 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:35 smithi083 systemd[1]: Started Ceph alertmanager.a for 7388281a-f4eb-11ee-b649-cb9ed24678a4. 2024-04-07T14:38:37.143 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:36 smithi083 ceph-mon[24810]: pgmap v67: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 667 B/s rd, 1.5 KiB/s wr, 2 op/s 2024-04-07T14:38:37.143 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:36 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:37.143 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:36 smithi083 ceph-mon[24810]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-07T14:38:37.143 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:36 smithi083 ceph-mon[24810]: Reconfiguring daemon prometheus.a on smithi189 2024-04-07T14:38:37.144 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:36 smithi083 ceph-mon[29774]: pgmap v67: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 667 B/s rd, 1.5 KiB/s wr, 2 op/s 2024-04-07T14:38:37.144 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:36 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:37.144 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:36 smithi083 ceph-mon[29774]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-07T14:38:37.144 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:36 smithi083 ceph-mon[29774]: Reconfiguring daemon prometheus.a on smithi189 2024-04-07T14:38:37.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:36 smithi189 ceph-mon[25797]: pgmap v67: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 667 B/s rd, 1.5 KiB/s wr, 2 op/s 2024-04-07T14:38:37.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:36 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:37.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:36 smithi189 ceph-mon[25797]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-07T14:38:37.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:36 smithi189 ceph-mon[25797]: Reconfiguring daemon prometheus.a on smithi189 2024-04-07T14:38:37.984 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:37 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=info ts=2024-04-07T14:38:37.694Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000659809s 2024-04-07T14:38:38.627 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:38:38 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:38:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:38:38.923 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:38 smithi189 ceph-mon[25797]: pgmap v68: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 682 B/s wr, 1 op/s 2024-04-07T14:38:38.924 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:38 smithi189 systemd[1]: Stopping Ceph prometheus.a for 7388281a-f4eb-11ee-b649-cb9ed24678a4... 2024-04-07T14:38:38.924 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:38 smithi189 bash[45534]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus.a" found: no such container 2024-04-07T14:38:39.089 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:38 smithi083 ceph-mon[24810]: pgmap v68: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 682 B/s wr, 1 op/s 2024-04-07T14:38:39.090 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:38 smithi083 ceph-mon[29774]: pgmap v68: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 597 B/s rd, 682 B/s wr, 1 op/s 2024-04-07T14:38:39.208 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:38 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[43522]: ts=2024-04-07T14:38:38.925Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-07T14:38:39.208 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:38 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[43522]: ts=2024-04-07T14:38:38.925Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-07T14:38:39.208 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:38 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[43522]: ts=2024-04-07T14:38:38.925Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-07T14:38:39.209 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:38 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[43522]: ts=2024-04-07T14:38:38.925Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-07T14:38:39.209 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:38 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[43522]: ts=2024-04-07T14:38:38.925Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-07T14:38:39.209 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:38 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[43522]: ts=2024-04-07T14:38:38.925Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-07T14:38:39.209 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:38 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[43522]: ts=2024-04-07T14:38:38.925Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-07T14:38:39.209 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:38 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[43522]: ts=2024-04-07T14:38:38.926Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-07T14:38:39.209 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:38 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[43522]: ts=2024-04-07T14:38:38.926Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-07T14:38:39.209 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:38 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[43522]: ts=2024-04-07T14:38:38.926Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-07T14:38:39.209 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:38 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[43522]: ts=2024-04-07T14:38:38.926Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-07T14:38:39.209 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:38 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[43522]: ts=2024-04-07T14:38:38.926Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-07T14:38:39.209 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:38 smithi189 podman[45547]: 2024-04-07 14:38:38.935834466 +0000 UTC m=+0.137355309 container died 19bf72cd5f3b217072f4aed2dd6c46aeb0ec85f6dfda16b2d1175a2b489a9dce (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-07T14:38:39.557 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:39 smithi189 podman[45547]: 2024-04-07 14:38:39.299681195 +0000 UTC m=+0.501202027 container cleanup 19bf72cd5f3b217072f4aed2dd6c46aeb0ec85f6dfda16b2d1175a2b489a9dce (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-07T14:38:39.557 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:39 smithi189 bash[45547]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a 2024-04-07T14:38:39.557 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:39 smithi189 bash[45575]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus.a" found: no such container 2024-04-07T14:38:39.816 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:39 smithi189 podman[45562]: 2024-04-07 14:38:39.559007117 +0000 UTC m=+0.624364610 container remove 19bf72cd5f3b217072f4aed2dd6c46aeb0ec85f6dfda16b2d1175a2b489a9dce (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-07T14:38:39.816 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:39 smithi189 systemd[1]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@prometheus.a.service: Deactivated successfully. 2024-04-07T14:38:39.816 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:39 smithi189 systemd[1]: Stopped Ceph prometheus.a for 7388281a-f4eb-11ee-b649-cb9ed24678a4. 2024-04-07T14:38:39.816 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:39 smithi189 systemd[1]: Starting Ceph prometheus.a for 7388281a-f4eb-11ee-b649-cb9ed24678a4... 2024-04-07T14:38:39.816 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:39 smithi189 podman[45647]: 2024-04-07 14:38:39.703321363 +0000 UTC m=+0.019740994 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-07T14:38:39.816 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:39 smithi189 podman[45647]: 2024-04-07 14:38:39.818208053 +0000 UTC m=+0.134627670 container create 1aeb972809bddb06ab7df776a72446e3bd027bcaee8bab6eb46a6d503953b91e (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-07T14:38:40.066 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:39 smithi189 ceph-mon[25797]: osdmap e64: 8 total, 8 up, 8 in 2024-04-07T14:38:40.067 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:39 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/3367333674' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-07T14:38:40.067 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:40 smithi189 podman[45647]: 2024-04-07 14:38:40.037540843 +0000 UTC m=+0.353960461 container init 1aeb972809bddb06ab7df776a72446e3bd027bcaee8bab6eb46a6d503953b91e (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-07T14:38:40.067 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:40 smithi189 podman[45647]: 2024-04-07 14:38:40.040191806 +0000 UTC m=+0.356611434 container start 1aeb972809bddb06ab7df776a72446e3bd027bcaee8bab6eb46a6d503953b91e (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-07T14:38:40.067 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:40 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[45663]: ts=2024-04-07T14:38:40.064Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-07T14:38:40.067 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:40 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[45663]: ts=2024-04-07T14:38:40.064Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-07T14:38:40.067 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:40 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[45663]: ts=2024-04-07T14:38:40.064Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-07T14:38:40.067 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:40 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[45663]: ts=2024-04-07T14:38:40.064Z caller=main.go:518 level=info host_details="(Linux 5.14.0-432.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Wed Mar 20 20:26:59 UTC 2024 x86_64 smithi189 (none))" 2024-04-07T14:38:40.067 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:40 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[45663]: ts=2024-04-07T14:38:40.064Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-07T14:38:40.067 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:40 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[45663]: ts=2024-04-07T14:38:40.064Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-07T14:38:40.067 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:40 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[45663]: ts=2024-04-07T14:38:40.066Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-07T14:38:40.067 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:40 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[45663]: ts=2024-04-07T14:38:40.066Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-07T14:38:40.068 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:40 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[45663]: ts=2024-04-07T14:38:40.067Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-07T14:38:40.068 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:40 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[45663]: ts=2024-04-07T14:38:40.068Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-07T14:38:40.068 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:40 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[45663]: ts=2024-04-07T14:38:40.068Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.416µs 2024-04-07T14:38:40.068 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:40 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[45663]: ts=2024-04-07T14:38:40.068Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-07T14:38:40.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:39 smithi083 ceph-mon[24810]: osdmap e64: 8 total, 8 up, 8 in 2024-04-07T14:38:40.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:39 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/3367333674' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-07T14:38:40.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:39 smithi083 ceph-mon[29774]: osdmap e64: 8 total, 8 up, 8 in 2024-04-07T14:38:40.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:39 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/3367333674' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-07T14:38:40.428 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:40 smithi189 bash[45647]: 1aeb972809bddb06ab7df776a72446e3bd027bcaee8bab6eb46a6d503953b91e 2024-04-07T14:38:40.429 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:40 smithi189 systemd[1]: Started Ceph prometheus.a for 7388281a-f4eb-11ee-b649-cb9ed24678a4. 2024-04-07T14:38:40.429 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:40 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[45663]: ts=2024-04-07T14:38:40.259Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-04-07T14:38:40.708 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:40 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[45663]: ts=2024-04-07T14:38:40.430Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-04-07T14:38:40.708 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:40 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[45663]: ts=2024-04-07T14:38:40.431Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-04-07T14:38:40.708 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:40 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[45663]: ts=2024-04-07T14:38:40.431Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=32.242µs wal_replay_duration=362.184049ms total_replay_duration=362.228497ms 2024-04-07T14:38:40.709 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:40 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[45663]: ts=2024-04-07T14:38:40.433Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-07T14:38:40.709 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:40 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[45663]: ts=2024-04-07T14:38:40.433Z caller=main.go:947 level=info msg="TSDB started" 2024-04-07T14:38:40.709 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:40 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[45663]: ts=2024-04-07T14:38:40.433Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-07T14:38:40.709 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:40 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[45663]: ts=2024-04-07T14:38:40.450Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=16.699171ms db_storage=766ns remote_storage=1.753µs web_handler=570ns query_engine=768ns scrape=581.867µs scrape_sd=45.196µs notify=38.062µs notify_sd=17.587µs rules=15.405871ms 2024-04-07T14:38:40.709 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:38:40 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[45663]: ts=2024-04-07T14:38:40.450Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-07T14:38:41.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:40 smithi083 ceph-mon[24810]: pgmap v70: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 716 B/s rd, 818 B/s wr, 1 op/s 2024-04-07T14:38:41.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:40 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/3367333674' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-04-07T14:38:41.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:40 smithi083 ceph-mon[24810]: osdmap e65: 8 total, 8 up, 8 in 2024-04-07T14:38:41.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:40 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:41.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:40 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-07T14:38:41.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:40 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.83:9093"}]: dispatch 2024-04-07T14:38:41.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:40 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:41.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:40 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-07T14:38:41.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:40 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-07T14:38:41.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:40 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.189:3000"}]: dispatch 2024-04-07T14:38:41.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:40 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:41.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:40 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:38:41.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:40 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:38:41.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:40 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:41.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:40 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:38:41.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:40 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:38:41.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:40 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:41.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:40 smithi083 ceph-mon[29774]: pgmap v70: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 716 B/s rd, 818 B/s wr, 1 op/s 2024-04-07T14:38:41.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:40 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/3367333674' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-04-07T14:38:41.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:40 smithi083 ceph-mon[29774]: osdmap e65: 8 total, 8 up, 8 in 2024-04-07T14:38:41.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:40 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:41.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:40 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-07T14:38:41.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:40 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.83:9093"}]: dispatch 2024-04-07T14:38:41.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:40 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:41.178 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:40 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-07T14:38:41.178 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:40 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-07T14:38:41.178 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:40 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.189:3000"}]: dispatch 2024-04-07T14:38:41.178 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:40 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:41.178 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:40 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:38:41.178 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:40 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:38:41.178 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:40 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:41.178 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:40 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:38:41.178 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:40 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:38:41.178 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:40 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:41.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:40 smithi189 ceph-mon[25797]: pgmap v70: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 716 B/s rd, 818 B/s wr, 1 op/s 2024-04-07T14:38:41.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:40 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/3367333674' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-04-07T14:38:41.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:40 smithi189 ceph-mon[25797]: osdmap e65: 8 total, 8 up, 8 in 2024-04-07T14:38:41.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:40 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:41.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:40 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-07T14:38:41.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:40 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.83:9093"}]: dispatch 2024-04-07T14:38:41.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:40 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:41.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:40 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-07T14:38:41.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:40 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-07T14:38:41.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:40 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.189:3000"}]: dispatch 2024-04-07T14:38:41.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:40 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:41.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:40 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:38:41.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:40 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:38:41.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:40 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:41.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:40 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:38:41.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:40 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:38:41.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:40 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:41.839 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:38:41 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:38:41] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:38:42.113 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:41 smithi189 ceph-mon[25797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-07T14:38:42.113 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:41 smithi189 ceph-mon[25797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.83:9093"}]: dispatch 2024-04-07T14:38:42.113 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:41 smithi189 ceph-mon[25797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-07T14:38:42.114 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:41 smithi189 ceph-mon[25797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-07T14:38:42.114 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:41 smithi189 ceph-mon[25797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.189:3000"}]: dispatch 2024-04-07T14:38:42.114 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:41 smithi189 ceph-mon[25797]: osdmap e66: 8 total, 8 up, 8 in 2024-04-07T14:38:42.114 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:41 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/3367333674' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-07T14:38:42.114 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:41 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:42.114 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:41 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:42.114 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:41 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/3367333674' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-04-07T14:38:42.114 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:41 smithi189 ceph-mon[25797]: osdmap e67: 8 total, 8 up, 8 in 2024-04-07T14:38:42.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:41 smithi083 ceph-mon[24810]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-07T14:38:42.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:41 smithi083 ceph-mon[24810]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.83:9093"}]: dispatch 2024-04-07T14:38:42.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:41 smithi083 ceph-mon[24810]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-07T14:38:42.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:41 smithi083 ceph-mon[24810]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-07T14:38:42.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:41 smithi083 ceph-mon[24810]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.189:3000"}]: dispatch 2024-04-07T14:38:42.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:41 smithi083 ceph-mon[24810]: osdmap e66: 8 total, 8 up, 8 in 2024-04-07T14:38:42.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:41 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/3367333674' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-07T14:38:42.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:41 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:42.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:41 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:42.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:41 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/3367333674' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-04-07T14:38:42.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:41 smithi083 ceph-mon[24810]: osdmap e67: 8 total, 8 up, 8 in 2024-04-07T14:38:42.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:41 smithi083 ceph-mon[29774]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-07T14:38:42.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:41 smithi083 ceph-mon[29774]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.83:9093"}]: dispatch 2024-04-07T14:38:42.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:41 smithi083 ceph-mon[29774]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-07T14:38:42.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:41 smithi083 ceph-mon[29774]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-07T14:38:42.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:41 smithi083 ceph-mon[29774]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.189:3000"}]: dispatch 2024-04-07T14:38:42.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:41 smithi083 ceph-mon[29774]: osdmap e66: 8 total, 8 up, 8 in 2024-04-07T14:38:42.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:41 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/3367333674' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-07T14:38:42.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:41 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:42.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:41 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:42.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:41 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/3367333674' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-04-07T14:38:42.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:41 smithi083 ceph-mon[29774]: osdmap e67: 8 total, 8 up, 8 in 2024-04-07T14:38:43.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:42 smithi083 ceph-mon[24810]: pgmap v73: 97 pgs: 32 unknown, 8 creating+activating, 57 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s rd, 3 op/s 2024-04-07T14:38:43.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:42 smithi083 ceph-mon[24810]: osdmap e68: 8 total, 8 up, 8 in 2024-04-07T14:38:43.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:42 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/1497529442' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-07T14:38:43.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:42 smithi083 ceph-mon[29774]: pgmap v73: 97 pgs: 32 unknown, 8 creating+activating, 57 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s rd, 3 op/s 2024-04-07T14:38:43.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:42 smithi083 ceph-mon[29774]: osdmap e68: 8 total, 8 up, 8 in 2024-04-07T14:38:43.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:42 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/1497529442' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-07T14:38:43.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:42 smithi189 ceph-mon[25797]: pgmap v73: 97 pgs: 32 unknown, 8 creating+activating, 57 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s rd, 3 op/s 2024-04-07T14:38:43.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:42 smithi189 ceph-mon[25797]: osdmap e68: 8 total, 8 up, 8 in 2024-04-07T14:38:43.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:42 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/1497529442' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-07T14:38:45.138 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:44 smithi189 ceph-mon[25797]: pgmap v76: 129 pgs: 64 unknown, 8 creating+activating, 57 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 5.7 KiB/s rd, 5 op/s 2024-04-07T14:38:45.138 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:44 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/1497529442' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-04-07T14:38:45.138 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:44 smithi189 ceph-mon[25797]: osdmap e69: 8 total, 8 up, 8 in 2024-04-07T14:38:45.138 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:44 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/1497529442' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-07T14:38:45.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:44 smithi083 ceph-mon[24810]: pgmap v76: 129 pgs: 64 unknown, 8 creating+activating, 57 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 5.7 KiB/s rd, 5 op/s 2024-04-07T14:38:45.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:44 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/1497529442' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-04-07T14:38:45.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:44 smithi083 ceph-mon[24810]: osdmap e69: 8 total, 8 up, 8 in 2024-04-07T14:38:45.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:44 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/1497529442' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-07T14:38:45.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:44 smithi083 ceph-mon[29774]: pgmap v76: 129 pgs: 64 unknown, 8 creating+activating, 57 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 5.7 KiB/s rd, 5 op/s 2024-04-07T14:38:45.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:44 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/1497529442' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-04-07T14:38:45.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:44 smithi083 ceph-mon[29774]: osdmap e69: 8 total, 8 up, 8 in 2024-04-07T14:38:45.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:44 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/1497529442' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-07T14:38:45.906 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-04-07T14:38:45.906 INFO:teuthology.orchestra.run.smithi083.stdout: "id": "85fa1cc8-d90e-4663-b083-fe896e587d25", 2024-04-07T14:38:45.906 INFO:teuthology.orchestra.run.smithi083.stdout: "epoch": 1, 2024-04-07T14:38:45.906 INFO:teuthology.orchestra.run.smithi083.stdout: "predecessor_uuid": "29108e7a-1a0f-4528-bc52-f38ccc4e2337", 2024-04-07T14:38:45.906 INFO:teuthology.orchestra.run.smithi083.stdout: "sync_status": [], 2024-04-07T14:38:45.907 INFO:teuthology.orchestra.run.smithi083.stdout: "period_map": { 2024-04-07T14:38:45.907 INFO:teuthology.orchestra.run.smithi083.stdout: "id": "85fa1cc8-d90e-4663-b083-fe896e587d25", 2024-04-07T14:38:45.907 INFO:teuthology.orchestra.run.smithi083.stdout: "zonegroups": [ 2024-04-07T14:38:45.907 INFO:teuthology.orchestra.run.smithi083.stdout: { 2024-04-07T14:38:45.907 INFO:teuthology.orchestra.run.smithi083.stdout: "id": "c91d3c32-b880-41b0-9802-f2466d027d40", 2024-04-07T14:38:45.907 INFO:teuthology.orchestra.run.smithi083.stdout: "name": "default", 2024-04-07T14:38:45.907 INFO:teuthology.orchestra.run.smithi083.stdout: "api_name": "default", 2024-04-07T14:38:45.907 INFO:teuthology.orchestra.run.smithi083.stdout: "is_master": "true", 2024-04-07T14:38:45.907 INFO:teuthology.orchestra.run.smithi083.stdout: "endpoints": [], 2024-04-07T14:38:45.907 INFO:teuthology.orchestra.run.smithi083.stdout: "hostnames": [], 2024-04-07T14:38:45.907 INFO:teuthology.orchestra.run.smithi083.stdout: "hostnames_s3website": [], 2024-04-07T14:38:45.907 INFO:teuthology.orchestra.run.smithi083.stdout: "master_zone": "f7959d84-9e92-4aa7-95a7-1c200af9e17f", 2024-04-07T14:38:45.907 INFO:teuthology.orchestra.run.smithi083.stdout: "zones": [ 2024-04-07T14:38:45.908 INFO:teuthology.orchestra.run.smithi083.stdout: { 2024-04-07T14:38:45.908 INFO:teuthology.orchestra.run.smithi083.stdout: "id": "f7959d84-9e92-4aa7-95a7-1c200af9e17f", 2024-04-07T14:38:45.908 INFO:teuthology.orchestra.run.smithi083.stdout: "name": "z", 2024-04-07T14:38:45.908 INFO:teuthology.orchestra.run.smithi083.stdout: "endpoints": [], 2024-04-07T14:38:45.908 INFO:teuthology.orchestra.run.smithi083.stdout: "log_meta": "false", 2024-04-07T14:38:45.908 INFO:teuthology.orchestra.run.smithi083.stdout: "log_data": "false", 2024-04-07T14:38:45.908 INFO:teuthology.orchestra.run.smithi083.stdout: "bucket_index_max_shards": 11, 2024-04-07T14:38:45.908 INFO:teuthology.orchestra.run.smithi083.stdout: "read_only": "false", 2024-04-07T14:38:45.908 INFO:teuthology.orchestra.run.smithi083.stdout: "tier_type": "", 2024-04-07T14:38:45.908 INFO:teuthology.orchestra.run.smithi083.stdout: "sync_from_all": "true", 2024-04-07T14:38:45.908 INFO:teuthology.orchestra.run.smithi083.stdout: "sync_from": [], 2024-04-07T14:38:45.908 INFO:teuthology.orchestra.run.smithi083.stdout: "redirect_zone": "" 2024-04-07T14:38:45.909 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-04-07T14:38:45.909 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-04-07T14:38:45.909 INFO:teuthology.orchestra.run.smithi083.stdout: "placement_targets": [ 2024-04-07T14:38:45.909 INFO:teuthology.orchestra.run.smithi083.stdout: { 2024-04-07T14:38:45.909 INFO:teuthology.orchestra.run.smithi083.stdout: "name": "default-placement", 2024-04-07T14:38:45.909 INFO:teuthology.orchestra.run.smithi083.stdout: "tags": [], 2024-04-07T14:38:45.909 INFO:teuthology.orchestra.run.smithi083.stdout: "storage_classes": [ 2024-04-07T14:38:45.909 INFO:teuthology.orchestra.run.smithi083.stdout: "STANDARD" 2024-04-07T14:38:45.909 INFO:teuthology.orchestra.run.smithi083.stdout: ] 2024-04-07T14:38:45.909 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-04-07T14:38:45.909 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-04-07T14:38:45.910 INFO:teuthology.orchestra.run.smithi083.stdout: "default_placement": "default-placement", 2024-04-07T14:38:45.910 INFO:teuthology.orchestra.run.smithi083.stdout: "realm_id": "3a89d6cc-37c5-47ad-85a8-f8c0285f9dde", 2024-04-07T14:38:45.910 INFO:teuthology.orchestra.run.smithi083.stdout: "sync_policy": { 2024-04-07T14:38:45.910 INFO:teuthology.orchestra.run.smithi083.stdout: "groups": [] 2024-04-07T14:38:45.910 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-04-07T14:38:45.910 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-04-07T14:38:45.910 INFO:teuthology.orchestra.run.smithi083.stdout: ], 2024-04-07T14:38:45.910 INFO:teuthology.orchestra.run.smithi083.stdout: "short_zone_ids": [ 2024-04-07T14:38:45.910 INFO:teuthology.orchestra.run.smithi083.stdout: { 2024-04-07T14:38:45.910 INFO:teuthology.orchestra.run.smithi083.stdout: "key": "f7959d84-9e92-4aa7-95a7-1c200af9e17f", 2024-04-07T14:38:45.910 INFO:teuthology.orchestra.run.smithi083.stdout: "val": 827612610 2024-04-07T14:38:45.910 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-04-07T14:38:45.910 INFO:teuthology.orchestra.run.smithi083.stdout: ] 2024-04-07T14:38:45.911 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-04-07T14:38:45.911 INFO:teuthology.orchestra.run.smithi083.stdout: "master_zonegroup": "c91d3c32-b880-41b0-9802-f2466d027d40", 2024-04-07T14:38:45.911 INFO:teuthology.orchestra.run.smithi083.stdout: "master_zone": "f7959d84-9e92-4aa7-95a7-1c200af9e17f", 2024-04-07T14:38:45.911 INFO:teuthology.orchestra.run.smithi083.stdout: "period_config": { 2024-04-07T14:38:45.911 INFO:teuthology.orchestra.run.smithi083.stdout: "bucket_quota": { 2024-04-07T14:38:45.911 INFO:teuthology.orchestra.run.smithi083.stdout: "enabled": false, 2024-04-07T14:38:45.911 INFO:teuthology.orchestra.run.smithi083.stdout: "check_on_raw": false, 2024-04-07T14:38:45.911 INFO:teuthology.orchestra.run.smithi083.stdout: "max_size": -1, 2024-04-07T14:38:45.911 INFO:teuthology.orchestra.run.smithi083.stdout: "max_size_kb": 0, 2024-04-07T14:38:45.911 INFO:teuthology.orchestra.run.smithi083.stdout: "max_objects": -1 2024-04-07T14:38:45.911 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-04-07T14:38:45.911 INFO:teuthology.orchestra.run.smithi083.stdout: "user_quota": { 2024-04-07T14:38:45.911 INFO:teuthology.orchestra.run.smithi083.stdout: "enabled": false, 2024-04-07T14:38:45.912 INFO:teuthology.orchestra.run.smithi083.stdout: "check_on_raw": false, 2024-04-07T14:38:45.912 INFO:teuthology.orchestra.run.smithi083.stdout: "max_size": -1, 2024-04-07T14:38:45.912 INFO:teuthology.orchestra.run.smithi083.stdout: "max_size_kb": 0, 2024-04-07T14:38:45.912 INFO:teuthology.orchestra.run.smithi083.stdout: "max_objects": -1 2024-04-07T14:38:45.912 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-04-07T14:38:45.912 INFO:teuthology.orchestra.run.smithi083.stdout: "user_ratelimit": { 2024-04-07T14:38:45.912 INFO:teuthology.orchestra.run.smithi083.stdout: "max_read_ops": 0, 2024-04-07T14:38:45.912 INFO:teuthology.orchestra.run.smithi083.stdout: "max_write_ops": 0, 2024-04-07T14:38:45.912 INFO:teuthology.orchestra.run.smithi083.stdout: "max_read_bytes": 0, 2024-04-07T14:38:45.912 INFO:teuthology.orchestra.run.smithi083.stdout: "max_write_bytes": 0, 2024-04-07T14:38:45.912 INFO:teuthology.orchestra.run.smithi083.stdout: "enabled": false 2024-04-07T14:38:45.912 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-04-07T14:38:45.912 INFO:teuthology.orchestra.run.smithi083.stdout: "bucket_ratelimit": { 2024-04-07T14:38:45.913 INFO:teuthology.orchestra.run.smithi083.stdout: "max_read_ops": 0, 2024-04-07T14:38:45.913 INFO:teuthology.orchestra.run.smithi083.stdout: "max_write_ops": 0, 2024-04-07T14:38:45.913 INFO:teuthology.orchestra.run.smithi083.stdout: "max_read_bytes": 0, 2024-04-07T14:38:45.913 INFO:teuthology.orchestra.run.smithi083.stdout: "max_write_bytes": 0, 2024-04-07T14:38:45.913 INFO:teuthology.orchestra.run.smithi083.stdout: "enabled": false 2024-04-07T14:38:45.913 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-04-07T14:38:45.913 INFO:teuthology.orchestra.run.smithi083.stdout: "anonymous_ratelimit": { 2024-04-07T14:38:45.913 INFO:teuthology.orchestra.run.smithi083.stdout: "max_read_ops": 0, 2024-04-07T14:38:45.913 INFO:teuthology.orchestra.run.smithi083.stdout: "max_write_ops": 0, 2024-04-07T14:38:45.913 INFO:teuthology.orchestra.run.smithi083.stdout: "max_read_bytes": 0, 2024-04-07T14:38:45.913 INFO:teuthology.orchestra.run.smithi083.stdout: "max_write_bytes": 0, 2024-04-07T14:38:45.913 INFO:teuthology.orchestra.run.smithi083.stdout: "enabled": false 2024-04-07T14:38:45.913 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-04-07T14:38:45.914 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-04-07T14:38:45.914 INFO:teuthology.orchestra.run.smithi083.stdout: "realm_id": "3a89d6cc-37c5-47ad-85a8-f8c0285f9dde", 2024-04-07T14:38:45.914 INFO:teuthology.orchestra.run.smithi083.stdout: "realm_name": "r", 2024-04-07T14:38:45.914 INFO:teuthology.orchestra.run.smithi083.stdout: "realm_epoch": 2 2024-04-07T14:38:45.915 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-04-07T14:38:46.119 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:45 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/1497529442' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-04-07T14:38:46.119 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:45 smithi189 ceph-mon[25797]: osdmap e70: 8 total, 8 up, 8 in 2024-04-07T14:38:46.119 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:45 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/1497529442' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-07T14:38:46.175 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:38:45 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=info ts=2024-04-07T14:38:45.697Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.003404205s 2024-04-07T14:38:46.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:45 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/1497529442' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-04-07T14:38:46.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:45 smithi083 ceph-mon[24810]: osdmap e70: 8 total, 8 up, 8 in 2024-04-07T14:38:46.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:45 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/1497529442' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-07T14:38:46.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:45 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/1497529442' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-04-07T14:38:46.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:45 smithi083 ceph-mon[29774]: osdmap e70: 8 total, 8 up, 8 in 2024-04-07T14:38:46.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:45 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/1497529442' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-07T14:38:46.646 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -e sha1=e5c885fe95ee982ff7b45214daa61b1a6d09d57d -- bash -c 'ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000' 2024-04-07T14:38:46.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:46 smithi083 ceph-mon[24810]: pgmap v79: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-04-07T14:38:46.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:46 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/1497529442' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-04-07T14:38:46.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:46 smithi083 ceph-mon[24810]: osdmap e71: 8 total, 8 up, 8 in 2024-04-07T14:38:46.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:46 smithi083 ceph-mon[29774]: pgmap v79: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-04-07T14:38:46.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:46 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/1497529442' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-04-07T14:38:46.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:46 smithi083 ceph-mon[29774]: osdmap e71: 8 total, 8 up, 8 in 2024-04-07T14:38:47.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:46 smithi189 ceph-mon[25797]: pgmap v79: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 511 B/s wr, 1 op/s 2024-04-07T14:38:47.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:46 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/1497529442' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-04-07T14:38:47.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:46 smithi189 ceph-mon[25797]: osdmap e71: 8 total, 8 up, 8 in 2024-04-07T14:38:48.708 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:38:48 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:38:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:38:48.787 INFO:teuthology.orchestra.run.smithi083.stdout:Scheduled rgw.foo update... 2024-04-07T14:38:49.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:48 smithi083 ceph-mon[24810]: pgmap v81: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 225 B/s rd, 451 B/s wr, 1 op/s 2024-04-07T14:38:49.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:48 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:49.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:48 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:38:49.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:48 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:38:49.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:48 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:49.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:48 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:49.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:48 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:49.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:48 smithi083 ceph-mon[29774]: pgmap v81: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 225 B/s rd, 451 B/s wr, 1 op/s 2024-04-07T14:38:49.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:48 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:49.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:48 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:38:49.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:48 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:38:49.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:48 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:49.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:48 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:49.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:48 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:49.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:48 smithi189 ceph-mon[25797]: pgmap v81: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 225 B/s rd, 451 B/s wr, 1 op/s 2024-04-07T14:38:49.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:48 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:49.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:48 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:38:49.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:48 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:38:49.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:48 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:49.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:48 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:49.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:48 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:49.575 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -e sha1=e5c885fe95ee982ff7b45214daa61b1a6d09d57d -- bash -c 'ceph osd pool create foo' 2024-04-07T14:38:50.131 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:49 smithi083 ceph-mon[29774]: from='client.24610 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:38:50.131 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:49 smithi083 ceph-mon[29774]: Saving service rgw.foo spec with placement count:2 2024-04-07T14:38:50.131 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:49 smithi083 ceph-mon[29774]: Saving service rgw.foo spec with placement count:2 2024-04-07T14:38:50.131 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:50.131 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi083.oslioj", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-07T14:38:50.131 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi083.oslioj", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-07T14:38:50.131 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:50.131 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:38:50.131 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:49 smithi083 ceph-mon[29774]: Deploying daemon rgw.foo.smithi083.oslioj on smithi083 2024-04-07T14:38:50.131 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1d", "id": [7, 2]}]: dispatch 2024-04-07T14:38:50.132 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2024-04-07T14:38:50.132 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-04-07T14:38:50.132 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 5]}]: dispatch 2024-04-07T14:38:50.132 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:38:50.132 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:38:50.132 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:49 smithi083 ceph-mon[24810]: from='client.24610 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:38:50.132 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:49 smithi083 ceph-mon[24810]: Saving service rgw.foo spec with placement count:2 2024-04-07T14:38:50.132 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:49 smithi083 ceph-mon[24810]: Saving service rgw.foo spec with placement count:2 2024-04-07T14:38:50.132 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:50.132 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi083.oslioj", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-07T14:38:50.132 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi083.oslioj", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-07T14:38:50.132 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:50.133 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:38:50.133 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:49 smithi083 ceph-mon[24810]: Deploying daemon rgw.foo.smithi083.oslioj on smithi083 2024-04-07T14:38:50.133 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1d", "id": [7, 2]}]: dispatch 2024-04-07T14:38:50.133 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2024-04-07T14:38:50.133 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-04-07T14:38:50.133 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 5]}]: dispatch 2024-04-07T14:38:50.133 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:38:50.133 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:38:50.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:49 smithi189 ceph-mon[25797]: from='client.24610 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:38:50.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:49 smithi189 ceph-mon[25797]: Saving service rgw.foo spec with placement count:2 2024-04-07T14:38:50.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:49 smithi189 ceph-mon[25797]: Saving service rgw.foo spec with placement count:2 2024-04-07T14:38:50.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:50.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi083.oslioj", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-07T14:38:50.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi083.oslioj", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-07T14:38:50.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:50.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:38:50.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:49 smithi189 ceph-mon[25797]: Deploying daemon rgw.foo.smithi083.oslioj on smithi083 2024-04-07T14:38:50.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1d", "id": [7, 2]}]: dispatch 2024-04-07T14:38:50.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2024-04-07T14:38:50.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-04-07T14:38:50.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 5]}]: dispatch 2024-04-07T14:38:50.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:38:50.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:38:51.172 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:50 smithi189 ceph-mon[25797]: pgmap v82: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-04-07T14:38:51.172 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:50 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1d", "id": [7, 2]}]': finished 2024-04-07T14:38:51.172 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:50 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2024-04-07T14:38:51.172 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:50 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-04-07T14:38:51.172 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:50 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 5]}]': finished 2024-04-07T14:38:51.172 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:50 smithi189 ceph-mon[25797]: osdmap e72: 8 total, 8 up, 8 in 2024-04-07T14:38:51.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:50 smithi083 ceph-mon[29774]: pgmap v82: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-04-07T14:38:51.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:50 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1d", "id": [7, 2]}]': finished 2024-04-07T14:38:51.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:50 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2024-04-07T14:38:51.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:50 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-04-07T14:38:51.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:50 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 5]}]': finished 2024-04-07T14:38:51.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:50 smithi083 ceph-mon[29774]: osdmap e72: 8 total, 8 up, 8 in 2024-04-07T14:38:51.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:50 smithi083 ceph-mon[24810]: pgmap v82: 129 pgs: 129 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-04-07T14:38:51.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:50 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1d", "id": [7, 2]}]': finished 2024-04-07T14:38:51.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:50 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2024-04-07T14:38:51.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:50 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-04-07T14:38:51.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:50 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 5]}]': finished 2024-04-07T14:38:51.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:50 smithi083 ceph-mon[24810]: osdmap e72: 8 total, 8 up, 8 in 2024-04-07T14:38:51.867 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:38:51 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:38:51] "GET /metrics HTTP/1.1" 200 196879 "" "Prometheus/2.33.4" 2024-04-07T14:38:52.175 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:51 smithi189 ceph-mon[25797]: osdmap e73: 8 total, 8 up, 8 in 2024-04-07T14:38:52.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:51 smithi083 ceph-mon[24810]: osdmap e73: 8 total, 8 up, 8 in 2024-04-07T14:38:52.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:51 smithi083 ceph-mon[29774]: osdmap e73: 8 total, 8 up, 8 in 2024-04-07T14:38:52.495 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:38:52 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:38:52] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:38:52.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:52 smithi189 ceph-mon[25797]: pgmap v85: 129 pgs: 4 peering, 125 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.2 KiB/s wr, 2 op/s 2024-04-07T14:38:52.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:52 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:52.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:52 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:53.145 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:52 smithi083 ceph-mon[24810]: pgmap v85: 129 pgs: 4 peering, 125 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.2 KiB/s wr, 2 op/s 2024-04-07T14:38:53.145 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:52 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:53.145 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:52 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:53.146 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:52 smithi083 ceph-mon[29774]: pgmap v85: 129 pgs: 4 peering, 125 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.2 KiB/s wr, 2 op/s 2024-04-07T14:38:53.146 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:52 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:53.146 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:52 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:53.912 INFO:teuthology.orchestra.run.smithi083.stderr:pool 'foo' created 2024-04-07T14:38:54.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:53 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/2525707060' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-07T14:38:54.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:53 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:54.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:53 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi189.taxdrv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-07T14:38:54.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:53 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi189.taxdrv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-07T14:38:54.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:53 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:54.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:53 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:38:54.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:53 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/2525707060' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-07T14:38:54.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:53 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:54.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:53 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi189.taxdrv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-07T14:38:54.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:53 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi189.taxdrv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-07T14:38:54.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:53 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:54.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:53 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:38:54.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:53 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/2525707060' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-07T14:38:54.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:53 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:54.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:53 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi189.taxdrv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-07T14:38:54.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:53 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi189.taxdrv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-07T14:38:54.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:53 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:38:54.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:53 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:38:54.655 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -e sha1=e5c885fe95ee982ff7b45214daa61b1a6d09d57d -- bash -c 'rbd pool init foo' 2024-04-07T14:38:54.916 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:54 smithi083 ceph-mon[24810]: pgmap v86: 129 pgs: 4 peering, 125 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 952 B/s rd, 952 B/s wr, 2 op/s 2024-04-07T14:38:54.916 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:54 smithi083 ceph-mon[24810]: Deploying daemon rgw.foo.smithi189.taxdrv on smithi189 2024-04-07T14:38:54.916 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:54 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/2525707060' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-04-07T14:38:54.916 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:54 smithi083 ceph-mon[24810]: osdmap e74: 8 total, 8 up, 8 in 2024-04-07T14:38:55.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:54 smithi083 ceph-mon[29774]: pgmap v86: 129 pgs: 4 peering, 125 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 952 B/s rd, 952 B/s wr, 2 op/s 2024-04-07T14:38:55.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:54 smithi083 ceph-mon[29774]: Deploying daemon rgw.foo.smithi189.taxdrv on smithi189 2024-04-07T14:38:55.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:54 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/2525707060' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-04-07T14:38:55.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:54 smithi083 ceph-mon[29774]: osdmap e74: 8 total, 8 up, 8 in 2024-04-07T14:38:55.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:54 smithi189 ceph-mon[25797]: pgmap v86: 129 pgs: 4 peering, 125 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 952 B/s rd, 952 B/s wr, 2 op/s 2024-04-07T14:38:55.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:54 smithi189 ceph-mon[25797]: Deploying daemon rgw.foo.smithi189.taxdrv on smithi189 2024-04-07T14:38:55.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:54 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/2525707060' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-04-07T14:38:55.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:54 smithi189 ceph-mon[25797]: osdmap e74: 8 total, 8 up, 8 in 2024-04-07T14:38:56.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:55 smithi083 ceph-mon[29774]: osdmap e75: 8 total, 8 up, 8 in 2024-04-07T14:38:56.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:55 smithi083 ceph-mon[24810]: osdmap e75: 8 total, 8 up, 8 in 2024-04-07T14:38:56.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:55 smithi189 ceph-mon[25797]: osdmap e75: 8 total, 8 up, 8 in 2024-04-07T14:38:57.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:56 smithi083 ceph-mon[24810]: pgmap v89: 161 pgs: 9 creating+peering, 23 creating+activating, 129 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s rd, 7.3 KiB/s wr, 207 op/s; 0 B/s, 0 objects/s recovering 2024-04-07T14:38:57.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:56 smithi083 ceph-mon[24810]: osdmap e76: 8 total, 8 up, 8 in 2024-04-07T14:38:57.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:56 smithi083 ceph-mon[29774]: pgmap v89: 161 pgs: 9 creating+peering, 23 creating+activating, 129 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s rd, 7.3 KiB/s wr, 207 op/s; 0 B/s, 0 objects/s recovering 2024-04-07T14:38:57.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:56 smithi083 ceph-mon[29774]: osdmap e76: 8 total, 8 up, 8 in 2024-04-07T14:38:57.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:56 smithi189 ceph-mon[25797]: pgmap v89: 161 pgs: 9 creating+peering, 23 creating+activating, 129 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s rd, 7.3 KiB/s wr, 207 op/s; 0 B/s, 0 objects/s recovering 2024-04-07T14:38:57.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:56 smithi189 ceph-mon[25797]: osdmap e76: 8 total, 8 up, 8 in 2024-04-07T14:38:58.708 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:38:58 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:38:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:38:59.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:58 smithi083 ceph-mon[24810]: pgmap v91: 161 pgs: 9 creating+peering, 23 creating+activating, 129 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 77 KiB/s rd, 5.5 KiB/s wr, 186 op/s; 0 B/s, 0 objects/s recovering 2024-04-07T14:38:59.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:58 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/1873338115' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-07T14:38:59.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:58 smithi083 ceph-mon[24810]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-07T14:38:59.177 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:38:58 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:38:58] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:38:59.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:58 smithi083 ceph-mon[29774]: pgmap v91: 161 pgs: 9 creating+peering, 23 creating+activating, 129 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 77 KiB/s rd, 5.5 KiB/s wr, 186 op/s; 0 B/s, 0 objects/s recovering 2024-04-07T14:38:59.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:58 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/1873338115' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-07T14:38:59.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:58 smithi083 ceph-mon[29774]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-07T14:38:59.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:58 smithi189 ceph-mon[25797]: pgmap v91: 161 pgs: 9 creating+peering, 23 creating+activating, 129 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 77 KiB/s rd, 5.5 KiB/s wr, 186 op/s; 0 B/s, 0 objects/s recovering 2024-04-07T14:38:59.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:58 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/1873338115' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-07T14:38:59.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:58 smithi189 ceph-mon[25797]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-07T14:39:00.072 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:59 smithi189 ceph-mon[25797]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-04-07T14:39:00.072 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:59 smithi189 ceph-mon[25797]: osdmap e77: 8 total, 8 up, 8 in 2024-04-07T14:39:00.072 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:59 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:00.072 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:38:59 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:00.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:59 smithi083 ceph-mon[29774]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-04-07T14:39:00.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:59 smithi083 ceph-mon[29774]: osdmap e77: 8 total, 8 up, 8 in 2024-04-07T14:39:00.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:59 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:00.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:38:59 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:00.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:59 smithi083 ceph-mon[24810]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-04-07T14:39:00.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:59 smithi083 ceph-mon[24810]: osdmap e77: 8 total, 8 up, 8 in 2024-04-07T14:39:00.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:59 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:00.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:38:59 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:01.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:00 smithi083 ceph-mon[24810]: pgmap v93: 161 pgs: 9 creating+peering, 23 creating+activating, 129 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 85 KiB/s rd, 6.0 KiB/s wr, 205 op/s; 0 B/s, 0 objects/s recovering 2024-04-07T14:39:01.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:00 smithi083 ceph-mon[24810]: osdmap e78: 8 total, 8 up, 8 in 2024-04-07T14:39:01.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:00 smithi083 ceph-mon[29774]: pgmap v93: 161 pgs: 9 creating+peering, 23 creating+activating, 129 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 85 KiB/s rd, 6.0 KiB/s wr, 205 op/s; 0 B/s, 0 objects/s recovering 2024-04-07T14:39:01.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:00 smithi083 ceph-mon[29774]: osdmap e78: 8 total, 8 up, 8 in 2024-04-07T14:39:01.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:00 smithi189 ceph-mon[25797]: pgmap v93: 161 pgs: 9 creating+peering, 23 creating+activating, 129 active+clean; 584 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 85 KiB/s rd, 6.0 KiB/s wr, 205 op/s; 0 B/s, 0 objects/s recovering 2024-04-07T14:39:01.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:00 smithi189 ceph-mon[25797]: osdmap e78: 8 total, 8 up, 8 in 2024-04-07T14:39:01.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:39:01] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:39:01.926 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:39:01] "GET /metrics HTTP/1.1" 200 196879 "" "Prometheus/2.33.4" 2024-04-07T14:39:01.926 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:39:01] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:39:02.131 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -e sha1=e5c885fe95ee982ff7b45214daa61b1a6d09d57d -- bash -c 'ceph orch apply iscsi foo u p' 2024-04-07T14:39:02.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:01 smithi189 ceph-mon[25797]: osdmap e79: 8 total, 8 up, 8 in 2024-04-07T14:39:02.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:01 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:02.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:01 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:39:02.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:01 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:39:02.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:01 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:02.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:01 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:02.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:01 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:02.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:01 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:39:02.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:01 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:39:02.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:01 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:02.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:01 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:02.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:01 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:02.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:01 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:39:02.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:01 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:39:02.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:01 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:02.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:01 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:02.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:01 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:39:02.210 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:01 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:39:02.210 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:01 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:02.210 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:01 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:02.237 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[24810]: osdmap e79: 8 total, 8 up, 8 in 2024-04-07T14:39:02.238 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:02.238 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:39:02.238 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:39:02.238 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:02.238 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:02.238 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:02.238 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:39:02.238 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:39:02.238 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:02.238 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:02.238 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:02.239 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:39:02.239 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:39:02.239 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:02.239 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:02.239 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:39:02.239 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:39:02.239 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:02.239 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:02.239 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[29774]: osdmap e79: 8 total, 8 up, 8 in 2024-04-07T14:39:02.239 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:02.239 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:39:02.239 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:39:02.240 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:02.240 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:02.240 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:02.240 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:39:02.240 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:39:02.240 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:02.240 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:02.240 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:02.240 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:39:02.240 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:39:02.240 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:02.241 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:02.241 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:39:02.241 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:39:02.241 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:02.241 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:01 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:03.285 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:39:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:39:03] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:39:03.286 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:03 smithi083 ceph-mon[24810]: Checking dashboard <-> RGW credentials 2024-04-07T14:39:03.286 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:03 smithi083 ceph-mon[24810]: pgmap v96: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:39:03.286 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:03 smithi083 ceph-mon[24810]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:39:03.286 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:03 smithi083 ceph-mon[24810]: Checking dashboard <-> RGW credentials 2024-04-07T14:39:03.286 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:03 smithi083 ceph-mon[24810]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:39:03.286 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:03 smithi083 ceph-mon[24810]: pgmap v97: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:39:03.286 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:03 smithi083 ceph-mon[24810]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:39:03.286 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:03 smithi083 ceph-mon[24810]: pgmap v98: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:39:03.287 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:03 smithi083 ceph-mon[24810]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:39:03.287 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:03 smithi083 ceph-mon[29774]: Checking dashboard <-> RGW credentials 2024-04-07T14:39:03.287 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:03 smithi083 ceph-mon[29774]: pgmap v96: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:39:03.287 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:03 smithi083 ceph-mon[29774]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:39:03.287 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:03 smithi083 ceph-mon[29774]: Checking dashboard <-> RGW credentials 2024-04-07T14:39:03.287 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:03 smithi083 ceph-mon[29774]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:39:03.287 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:03 smithi083 ceph-mon[29774]: pgmap v97: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:39:03.287 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:03 smithi083 ceph-mon[29774]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:39:03.287 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:03 smithi083 ceph-mon[29774]: pgmap v98: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:39:03.287 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:03 smithi083 ceph-mon[29774]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:39:03.297 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:03 smithi189 ceph-mon[25797]: Checking dashboard <-> RGW credentials 2024-04-07T14:39:03.297 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:03 smithi189 ceph-mon[25797]: pgmap v96: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:39:03.297 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:03 smithi189 ceph-mon[25797]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:39:03.297 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:03 smithi189 ceph-mon[25797]: Checking dashboard <-> RGW credentials 2024-04-07T14:39:03.297 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:03 smithi189 ceph-mon[25797]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:39:03.297 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:03 smithi189 ceph-mon[25797]: pgmap v97: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:39:03.297 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:03 smithi189 ceph-mon[25797]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:39:03.298 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:03 smithi189 ceph-mon[25797]: pgmap v98: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:39:03.298 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:03 smithi189 ceph-mon[25797]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:39:04.301 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:04 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:04.301 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:04 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:04.301 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:04 smithi189 ceph-mon[25797]: pgmap v99: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:39:04.301 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:04 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:04.518 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:04 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:04.519 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:04 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:04.519 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:04 smithi083 ceph-mon[24810]: pgmap v99: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:39:04.519 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:04 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:04.519 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:04 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:04.519 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:04 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:04.519 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:04 smithi083 ceph-mon[29774]: pgmap v99: 161 pgs: 161 active+clean; 584 KiB data, 55 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:39:04.519 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:04 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:05.451 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:39:05 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:39:05] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:39:05.999 INFO:teuthology.orchestra.run.smithi083.stdout:Scheduled iscsi.foo update... 2024-04-07T14:39:06.420 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:06 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:06.420 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:06 smithi083 ceph-mon[24810]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 125 KiB/s rd, 886 B/s wr, 187 op/s 2024-04-07T14:39:06.420 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:06 smithi083 ceph-mon[24810]: from='client.24742 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:39:06.420 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:06 smithi083 ceph-mon[24810]: Saving service iscsi.foo spec with placement count:1 2024-04-07T14:39:06.420 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:06 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:06.420 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:06 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:39:06.420 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:06 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:39:06.420 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:06 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:06.420 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:06 smithi083 ceph-mon[24810]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:39:06.421 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:06 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:06.421 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:06 smithi083 ceph-mon[29774]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 125 KiB/s rd, 886 B/s wr, 187 op/s 2024-04-07T14:39:06.421 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:06 smithi083 ceph-mon[29774]: from='client.24742 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:39:06.421 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:06 smithi083 ceph-mon[29774]: Saving service iscsi.foo spec with placement count:1 2024-04-07T14:39:06.421 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:06 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:06.421 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:06 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:39:06.421 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:06 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:39:06.421 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:06 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:06.421 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:06 smithi083 ceph-mon[29774]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:39:06.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:06 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:06.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:06 smithi189 ceph-mon[25797]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 125 KiB/s rd, 886 B/s wr, 187 op/s 2024-04-07T14:39:06.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:06 smithi189 ceph-mon[25797]: from='client.24742 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:39:06.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:06 smithi189 ceph-mon[25797]: Saving service iscsi.foo spec with placement count:1 2024-04-07T14:39:06.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:06 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:06.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:06 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:39:06.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:06 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:39:06.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:06 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:06.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:06 smithi189 ceph-mon[25797]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:39:06.843 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -e sha1=e5c885fe95ee982ff7b45214daa61b1a6d09d57d -- bash -c 'sleep 180' 2024-04-07T14:39:07.675 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:39:07 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:39:07] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:39:08.019 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:39:07 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:39:07] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:39:08.503 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:39:08 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:39:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:39:08.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:08 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:08.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:08 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:08.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:08 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:08.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:08 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:08.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:08 smithi083 ceph-mon[24810]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 106 KiB/s rd, 756 B/s wr, 159 op/s 2024-04-07T14:39:08.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:08 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:39:08.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:08 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:39:08.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:08 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:08.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:08 smithi083 ceph-mon[24810]: Checking pool "foo" exists for service iscsi.foo 2024-04-07T14:39:08.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:08 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi083.qmxyme", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-07T14:39:08.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:08 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi083.qmxyme", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-04-07T14:39:08.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:08 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:39:08.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:08 smithi083 ceph-mon[24810]: Deploying daemon iscsi.foo.smithi083.qmxyme on smithi083 2024-04-07T14:39:08.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:08 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:08.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:08 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:08.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:08 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:08.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:08 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:08.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:08 smithi083 ceph-mon[29774]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 106 KiB/s rd, 756 B/s wr, 159 op/s 2024-04-07T14:39:08.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:08 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:39:08.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:08 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:39:08.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:08 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:08.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:08 smithi083 ceph-mon[29774]: Checking pool "foo" exists for service iscsi.foo 2024-04-07T14:39:08.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:08 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi083.qmxyme", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-07T14:39:08.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:08 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi083.qmxyme", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-04-07T14:39:08.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:08 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:39:08.678 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:08 smithi083 ceph-mon[29774]: Deploying daemon iscsi.foo.smithi083.qmxyme on smithi083 2024-04-07T14:39:08.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:08 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:08.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:08 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:08.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:08 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:08.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:08 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:08.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:08 smithi189 ceph-mon[25797]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 106 KiB/s rd, 756 B/s wr, 159 op/s 2024-04-07T14:39:08.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:08 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:39:08.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:08 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:39:08.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:08 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:08.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:08 smithi189 ceph-mon[25797]: Checking pool "foo" exists for service iscsi.foo 2024-04-07T14:39:08.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:08 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi083.qmxyme", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-07T14:39:08.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:08 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi083.qmxyme", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-04-07T14:39:08.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:08 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:39:08.959 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:08 smithi189 ceph-mon[25797]: Deploying daemon iscsi.foo.smithi083.qmxyme on smithi083 2024-04-07T14:39:10.771 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:10 smithi083 ceph-mon[24810]: mgrmap e20: y(active, since 2m), standbys: x 2024-04-07T14:39:10.771 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:10 smithi083 ceph-mon[24810]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s rd, 612 B/s wr, 129 op/s 2024-04-07T14:39:10.771 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:10 smithi083 ceph-mon[24810]: osdmap e80: 8 total, 8 up, 8 in 2024-04-07T14:39:10.771 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:10 smithi083 ceph-mon[29774]: mgrmap e20: y(active, since 2m), standbys: x 2024-04-07T14:39:10.772 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:10 smithi083 ceph-mon[29774]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s rd, 612 B/s wr, 129 op/s 2024-04-07T14:39:10.772 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:10 smithi083 ceph-mon[29774]: osdmap e80: 8 total, 8 up, 8 in 2024-04-07T14:39:10.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:10 smithi189 ceph-mon[25797]: mgrmap e20: y(active, since 2m), standbys: x 2024-04-07T14:39:10.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:10 smithi189 ceph-mon[25797]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s rd, 612 B/s wr, 129 op/s 2024-04-07T14:39:10.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:10 smithi189 ceph-mon[25797]: osdmap e80: 8 total, 8 up, 8 in 2024-04-07T14:39:11.966 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:39:11 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:39:11] "GET /metrics HTTP/1.1" 200 207242 "" "Prometheus/2.33.4" 2024-04-07T14:39:13.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[29774]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 511 B/s wr, 108 op/s 2024-04-07T14:39:13.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:13.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:39:13.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:39:13.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:13.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[29774]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:39:13.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-07T14:39:13.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[29774]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-07T14:39:13.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[29774]: Adding iSCSI gateway http://:@172.21.15.83:5000 to Dashboard 2024-04-07T14:39:13.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-07T14:39:13.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[29774]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-07T14:39:13.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:13.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi083"}]: dispatch 2024-04-07T14:39:13.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:13.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:39:13.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:39:13.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:13.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/2136942724' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-07T14:39:13.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/1683190048' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/248923343"}]: dispatch 2024-04-07T14:39:13.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[29774]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/248923343"}]: dispatch 2024-04-07T14:39:13.177 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[24810]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 511 B/s wr, 108 op/s 2024-04-07T14:39:13.178 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:13.178 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:39:13.178 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:39:13.178 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:13.178 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[24810]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:39:13.178 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-07T14:39:13.178 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[24810]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-07T14:39:13.178 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[24810]: Adding iSCSI gateway http://:@172.21.15.83:5000 to Dashboard 2024-04-07T14:39:13.178 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-07T14:39:13.178 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[24810]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-07T14:39:13.178 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:13.178 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi083"}]: dispatch 2024-04-07T14:39:13.178 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:13.178 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:39:13.179 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:39:13.179 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:13.179 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/2136942724' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-07T14:39:13.179 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/1683190048' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/248923343"}]: dispatch 2024-04-07T14:39:13.179 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:13 smithi083 ceph-mon[24810]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/248923343"}]: dispatch 2024-04-07T14:39:13.365 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:13 smithi189 ceph-mon[25797]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 511 B/s wr, 108 op/s 2024-04-07T14:39:13.365 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:13 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:13.365 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:13 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:39:13.365 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:13 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:39:13.365 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:13 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:13.366 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:13 smithi189 ceph-mon[25797]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:39:13.366 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:13 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-07T14:39:13.366 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:13 smithi189 ceph-mon[25797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-07T14:39:13.366 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:13 smithi189 ceph-mon[25797]: Adding iSCSI gateway http://:@172.21.15.83:5000 to Dashboard 2024-04-07T14:39:13.366 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:13 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-07T14:39:13.366 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:13 smithi189 ceph-mon[25797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-07T14:39:13.366 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:13 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:13.366 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:13 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi083"}]: dispatch 2024-04-07T14:39:13.366 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:13 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:13.366 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:13 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:39:13.366 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:13 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:39:13.366 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:13 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:13.366 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:13 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/2136942724' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-07T14:39:13.366 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:13 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/1683190048' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/248923343"}]: dispatch 2024-04-07T14:39:13.367 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:13 smithi189 ceph-mon[25797]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/248923343"}]: dispatch 2024-04-07T14:39:14.292 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:14 smithi083 ceph-mon[24810]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi083"}]: dispatch 2024-04-07T14:39:14.293 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:14 smithi083 ceph-mon[24810]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:39:14.293 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:14 smithi083 ceph-mon[24810]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/248923343"}]': finished 2024-04-07T14:39:14.293 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:14 smithi083 ceph-mon[24810]: osdmap e81: 8 total, 8 up, 8 in 2024-04-07T14:39:14.293 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:14 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/3440799545' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/907073394"}]: dispatch 2024-04-07T14:39:14.293 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:14 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:14.293 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:14 smithi083 ceph-mon[29774]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi083"}]: dispatch 2024-04-07T14:39:14.293 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:14 smithi083 ceph-mon[29774]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:39:14.293 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:14 smithi083 ceph-mon[29774]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/248923343"}]': finished 2024-04-07T14:39:14.293 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:14 smithi083 ceph-mon[29774]: osdmap e81: 8 total, 8 up, 8 in 2024-04-07T14:39:14.294 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:14 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/3440799545' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/907073394"}]: dispatch 2024-04-07T14:39:14.294 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:14 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:14.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:14 smithi189 ceph-mon[25797]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi083"}]: dispatch 2024-04-07T14:39:14.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:14 smithi189 ceph-mon[25797]: Metadata not up to date on all hosts. Skipping non agent specs 2024-04-07T14:39:14.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:14 smithi189 ceph-mon[25797]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/248923343"}]': finished 2024-04-07T14:39:14.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:14 smithi189 ceph-mon[25797]: osdmap e81: 8 total, 8 up, 8 in 2024-04-07T14:39:14.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:14 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/3440799545' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/907073394"}]: dispatch 2024-04-07T14:39:14.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:14 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:15.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:15 smithi083 ceph-mon[24810]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:39:15.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:15 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/3440799545' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/907073394"}]': finished 2024-04-07T14:39:15.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:15 smithi083 ceph-mon[24810]: osdmap e82: 8 total, 8 up, 8 in 2024-04-07T14:39:15.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:15 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/885449343' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6800/2425895246"}]: dispatch 2024-04-07T14:39:15.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:15 smithi083 ceph-mon[29774]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:39:15.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:15 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/3440799545' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/907073394"}]': finished 2024-04-07T14:39:15.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:15 smithi083 ceph-mon[29774]: osdmap e82: 8 total, 8 up, 8 in 2024-04-07T14:39:15.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:15 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/885449343' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6800/2425895246"}]: dispatch 2024-04-07T14:39:15.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:15 smithi189 ceph-mon[25797]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:39:15.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:15 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/3440799545' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/907073394"}]': finished 2024-04-07T14:39:15.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:15 smithi189 ceph-mon[25797]: osdmap e82: 8 total, 8 up, 8 in 2024-04-07T14:39:15.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:15 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/885449343' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6800/2425895246"}]: dispatch 2024-04-07T14:39:16.287 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:39:16 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:39:16] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:39:16.287 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:16 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/885449343' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6800/2425895246"}]': finished 2024-04-07T14:39:16.287 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:16 smithi083 ceph-mon[24810]: osdmap e83: 8 total, 8 up, 8 in 2024-04-07T14:39:16.287 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:16 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/778496639' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/482476659"}]: dispatch 2024-04-07T14:39:16.287 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:16 smithi083 ceph-mon[24810]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/482476659"}]: dispatch 2024-04-07T14:39:16.287 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:16 smithi083 ceph-mon[24810]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 562 B/s rd, 187 B/s wr, 2 op/s 2024-04-07T14:39:16.288 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:16 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:16.288 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:16 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/885449343' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6800/2425895246"}]': finished 2024-04-07T14:39:16.288 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:16 smithi083 ceph-mon[29774]: osdmap e83: 8 total, 8 up, 8 in 2024-04-07T14:39:16.288 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:16 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/778496639' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/482476659"}]: dispatch 2024-04-07T14:39:16.288 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:16 smithi083 ceph-mon[29774]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/482476659"}]: dispatch 2024-04-07T14:39:16.288 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:16 smithi083 ceph-mon[29774]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 562 B/s rd, 187 B/s wr, 2 op/s 2024-04-07T14:39:16.288 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:16 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:16.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:16 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/885449343' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6800/2425895246"}]': finished 2024-04-07T14:39:16.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:16 smithi189 ceph-mon[25797]: osdmap e83: 8 total, 8 up, 8 in 2024-04-07T14:39:16.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:16 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/778496639' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/482476659"}]: dispatch 2024-04-07T14:39:16.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:16 smithi189 ceph-mon[25797]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/482476659"}]: dispatch 2024-04-07T14:39:16.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:16 smithi189 ceph-mon[25797]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 562 B/s rd, 187 B/s wr, 2 op/s 2024-04-07T14:39:16.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:16 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:17.293 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:17 smithi083 ceph-mon[24810]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/482476659"}]': finished 2024-04-07T14:39:17.293 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:17 smithi083 ceph-mon[24810]: osdmap e84: 8 total, 8 up, 8 in 2024-04-07T14:39:17.293 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:17 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/2790454951' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/1572934944"}]: dispatch 2024-04-07T14:39:17.293 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:17 smithi083 ceph-mon[29774]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/482476659"}]': finished 2024-04-07T14:39:17.293 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:17 smithi083 ceph-mon[29774]: osdmap e84: 8 total, 8 up, 8 in 2024-04-07T14:39:17.293 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:17 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/2790454951' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/1572934944"}]: dispatch 2024-04-07T14:39:17.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:17 smithi189 ceph-mon[25797]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/482476659"}]': finished 2024-04-07T14:39:17.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:17 smithi189 ceph-mon[25797]: osdmap e84: 8 total, 8 up, 8 in 2024-04-07T14:39:17.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:17 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/2790454951' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/1572934944"}]: dispatch 2024-04-07T14:39:18.536 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:18 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/2790454951' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/1572934944"}]': finished 2024-04-07T14:39:18.536 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:18 smithi083 ceph-mon[24810]: osdmap e85: 8 total, 8 up, 8 in 2024-04-07T14:39:18.536 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:18 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/292361254' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/848305012"}]: dispatch 2024-04-07T14:39:18.536 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:18 smithi083 ceph-mon[24810]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 255 B/s wr, 2 op/s 2024-04-07T14:39:18.536 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:18 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/2790454951' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/1572934944"}]': finished 2024-04-07T14:39:18.537 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:18 smithi083 ceph-mon[29774]: osdmap e85: 8 total, 8 up, 8 in 2024-04-07T14:39:18.537 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:18 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/292361254' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/848305012"}]: dispatch 2024-04-07T14:39:18.537 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:18 smithi083 ceph-mon[29774]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 255 B/s wr, 2 op/s 2024-04-07T14:39:18.708 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:39:18 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:39:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:39:18.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:18 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/2790454951' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/1572934944"}]': finished 2024-04-07T14:39:18.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:18 smithi189 ceph-mon[25797]: osdmap e85: 8 total, 8 up, 8 in 2024-04-07T14:39:18.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:18 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/292361254' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/848305012"}]: dispatch 2024-04-07T14:39:18.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:18 smithi189 ceph-mon[25797]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 255 B/s wr, 2 op/s 2024-04-07T14:39:19.280 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:39:18 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:39:18] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:39:19.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:19 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/292361254' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/848305012"}]': finished 2024-04-07T14:39:19.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:19 smithi083 ceph-mon[24810]: osdmap e86: 8 total, 8 up, 8 in 2024-04-07T14:39:19.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:19 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/2142737659' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6801/2425895246"}]: dispatch 2024-04-07T14:39:19.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:19 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:19.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:19 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:19.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:19 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:39:19.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:19 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:39:19.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:19 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:19.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:19 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/292361254' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/848305012"}]': finished 2024-04-07T14:39:19.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:19 smithi083 ceph-mon[29774]: osdmap e86: 8 total, 8 up, 8 in 2024-04-07T14:39:19.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:19 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/2142737659' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6801/2425895246"}]: dispatch 2024-04-07T14:39:19.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:19 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:19.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:19 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:19.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:19 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:39:19.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:19 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:39:19.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:19 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:19.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:19 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/292361254' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/848305012"}]': finished 2024-04-07T14:39:19.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:19 smithi189 ceph-mon[25797]: osdmap e86: 8 total, 8 up, 8 in 2024-04-07T14:39:19.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:19 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/2142737659' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6801/2425895246"}]: dispatch 2024-04-07T14:39:19.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:19 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:19.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:19 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:19.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:19 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:39:19.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:19 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:39:19.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:19 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:20.664 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:20 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/2142737659' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6801/2425895246"}]': finished 2024-04-07T14:39:20.665 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:20 smithi189 ceph-mon[25797]: osdmap e87: 8 total, 8 up, 8 in 2024-04-07T14:39:20.665 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:20 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/3258198176' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6800/3303842870"}]: dispatch 2024-04-07T14:39:20.665 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:20 smithi189 ceph-mon[25797]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6800/3303842870"}]: dispatch 2024-04-07T14:39:20.665 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:20 smithi189 ceph-mon[25797]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:39:20.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:20 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/2142737659' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6801/2425895246"}]': finished 2024-04-07T14:39:20.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:20 smithi083 ceph-mon[24810]: osdmap e87: 8 total, 8 up, 8 in 2024-04-07T14:39:20.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:20 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/3258198176' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6800/3303842870"}]: dispatch 2024-04-07T14:39:20.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:20 smithi083 ceph-mon[24810]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6800/3303842870"}]: dispatch 2024-04-07T14:39:20.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:20 smithi083 ceph-mon[24810]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:39:20.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:20 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/2142737659' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6801/2425895246"}]': finished 2024-04-07T14:39:20.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:20 smithi083 ceph-mon[29774]: osdmap e87: 8 total, 8 up, 8 in 2024-04-07T14:39:20.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:20 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/3258198176' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6800/3303842870"}]: dispatch 2024-04-07T14:39:20.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:20 smithi083 ceph-mon[29774]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6800/3303842870"}]: dispatch 2024-04-07T14:39:20.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:20 smithi083 ceph-mon[29774]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:39:21.175 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:39:20 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:39:20] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:39:21.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:21 smithi083 ceph-mon[24810]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6800/3303842870"}]': finished 2024-04-07T14:39:21.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:21 smithi083 ceph-mon[24810]: osdmap e88: 8 total, 8 up, 8 in 2024-04-07T14:39:21.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:21 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/280091644' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/3248703110"}]: dispatch 2024-04-07T14:39:21.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:21 smithi083 ceph-mon[24810]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/3248703110"}]: dispatch 2024-04-07T14:39:21.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:21 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:21.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:21 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:21.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:21 smithi083 ceph-mon[29774]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6800/3303842870"}]': finished 2024-04-07T14:39:21.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:21 smithi083 ceph-mon[29774]: osdmap e88: 8 total, 8 up, 8 in 2024-04-07T14:39:21.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:21 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/280091644' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/3248703110"}]: dispatch 2024-04-07T14:39:21.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:21 smithi083 ceph-mon[29774]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/3248703110"}]: dispatch 2024-04-07T14:39:21.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:21 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:21.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:21 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:21.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:21 smithi189 ceph-mon[25797]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6800/3303842870"}]': finished 2024-04-07T14:39:21.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:21 smithi189 ceph-mon[25797]: osdmap e88: 8 total, 8 up, 8 in 2024-04-07T14:39:21.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:21 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/280091644' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/3248703110"}]: dispatch 2024-04-07T14:39:21.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:21 smithi189 ceph-mon[25797]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/3248703110"}]: dispatch 2024-04-07T14:39:21.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:21 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:21.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:21 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:22.175 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:39:21 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:39:21] "GET /metrics HTTP/1.1" 200 208138 "" "Prometheus/2.33.4" 2024-04-07T14:39:22.585 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:22 smithi189 ceph-mon[25797]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/3248703110"}]': finished 2024-04-07T14:39:22.585 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:22 smithi189 ceph-mon[25797]: osdmap e89: 8 total, 8 up, 8 in 2024-04-07T14:39:22.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:22 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/4048736090' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/1472449846"}]: dispatch 2024-04-07T14:39:22.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:22 smithi189 ceph-mon[25797]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/1472449846"}]: dispatch 2024-04-07T14:39:22.586 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:22 smithi189 ceph-mon[25797]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:39:22.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:22 smithi083 ceph-mon[29774]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/3248703110"}]': finished 2024-04-07T14:39:22.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:22 smithi083 ceph-mon[29774]: osdmap e89: 8 total, 8 up, 8 in 2024-04-07T14:39:22.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:22 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/4048736090' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/1472449846"}]: dispatch 2024-04-07T14:39:22.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:22 smithi083 ceph-mon[29774]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/1472449846"}]: dispatch 2024-04-07T14:39:22.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:22 smithi083 ceph-mon[29774]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:39:22.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:22 smithi083 ceph-mon[24810]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/3248703110"}]': finished 2024-04-07T14:39:22.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:22 smithi083 ceph-mon[24810]: osdmap e89: 8 total, 8 up, 8 in 2024-04-07T14:39:22.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:22 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/4048736090' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/1472449846"}]: dispatch 2024-04-07T14:39:22.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:22 smithi083 ceph-mon[24810]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/1472449846"}]: dispatch 2024-04-07T14:39:22.677 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:22 smithi083 ceph-mon[24810]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:39:23.575 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:23 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:39:23.575 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:23 smithi189 ceph-mon[25797]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/1472449846"}]': finished 2024-04-07T14:39:23.575 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:23 smithi189 ceph-mon[25797]: osdmap e90: 8 total, 8 up, 8 in 2024-04-07T14:39:23.575 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:23 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/148504022' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6800/533197939"}]: dispatch 2024-04-07T14:39:23.575 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:23 smithi189 ceph-mon[25797]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6800/533197939"}]: dispatch 2024-04-07T14:39:23.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:23 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:39:23.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:23 smithi083 ceph-mon[24810]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/1472449846"}]': finished 2024-04-07T14:39:23.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:23 smithi083 ceph-mon[24810]: osdmap e90: 8 total, 8 up, 8 in 2024-04-07T14:39:23.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:23 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/148504022' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6800/533197939"}]: dispatch 2024-04-07T14:39:23.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:23 smithi083 ceph-mon[24810]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6800/533197939"}]: dispatch 2024-04-07T14:39:23.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:23 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:39:23.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:23 smithi083 ceph-mon[29774]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/1472449846"}]': finished 2024-04-07T14:39:23.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:23 smithi083 ceph-mon[29774]: osdmap e90: 8 total, 8 up, 8 in 2024-04-07T14:39:23.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:23 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/148504022' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6800/533197939"}]: dispatch 2024-04-07T14:39:23.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:23 smithi083 ceph-mon[29774]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6800/533197939"}]: dispatch 2024-04-07T14:39:24.675 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:24 smithi083 ceph-mon[29774]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6800/533197939"}]': finished 2024-04-07T14:39:24.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:24 smithi083 ceph-mon[29774]: osdmap e91: 8 total, 8 up, 8 in 2024-04-07T14:39:24.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:24 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/3169446704' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6801/3303842870"}]: dispatch 2024-04-07T14:39:24.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:24 smithi083 ceph-mon[29774]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:39:24.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:24 smithi083 ceph-mon[24810]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6800/533197939"}]': finished 2024-04-07T14:39:24.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:24 smithi083 ceph-mon[24810]: osdmap e91: 8 total, 8 up, 8 in 2024-04-07T14:39:24.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:24 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/3169446704' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6801/3303842870"}]: dispatch 2024-04-07T14:39:24.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:24 smithi083 ceph-mon[24810]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:39:24.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:24 smithi189 ceph-mon[25797]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6800/533197939"}]': finished 2024-04-07T14:39:24.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:24 smithi189 ceph-mon[25797]: osdmap e91: 8 total, 8 up, 8 in 2024-04-07T14:39:24.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:24 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/3169446704' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6801/3303842870"}]: dispatch 2024-04-07T14:39:24.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:24 smithi189 ceph-mon[25797]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:39:25.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:25 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/3169446704' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6801/3303842870"}]': finished 2024-04-07T14:39:25.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:25 smithi083 ceph-mon[24810]: osdmap e92: 8 total, 8 up, 8 in 2024-04-07T14:39:25.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:25 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/3500537032' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6801/533197939"}]: dispatch 2024-04-07T14:39:25.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:25 smithi083 ceph-mon[24810]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6801/533197939"}]: dispatch 2024-04-07T14:39:25.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:25 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/3169446704' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6801/3303842870"}]': finished 2024-04-07T14:39:25.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:25 smithi083 ceph-mon[29774]: osdmap e92: 8 total, 8 up, 8 in 2024-04-07T14:39:25.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:25 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/3500537032' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6801/533197939"}]: dispatch 2024-04-07T14:39:25.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:25 smithi083 ceph-mon[29774]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6801/533197939"}]: dispatch 2024-04-07T14:39:25.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:25 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/3169446704' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6801/3303842870"}]': finished 2024-04-07T14:39:25.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:25 smithi189 ceph-mon[25797]: osdmap e92: 8 total, 8 up, 8 in 2024-04-07T14:39:25.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:25 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/3500537032' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6801/533197939"}]: dispatch 2024-04-07T14:39:25.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:25 smithi189 ceph-mon[25797]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6801/533197939"}]: dispatch 2024-04-07T14:39:26.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:26 smithi083 ceph-mon[24810]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6801/533197939"}]': finished 2024-04-07T14:39:26.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:26 smithi083 ceph-mon[24810]: osdmap e93: 8 total, 8 up, 8 in 2024-04-07T14:39:26.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:26 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/3400244154' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/145418573"}]: dispatch 2024-04-07T14:39:26.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:26 smithi083 ceph-mon[24810]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/145418573"}]: dispatch 2024-04-07T14:39:26.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:26 smithi083 ceph-mon[24810]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:39:26.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:26 smithi083 ceph-mon[29774]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6801/533197939"}]': finished 2024-04-07T14:39:26.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:26 smithi083 ceph-mon[29774]: osdmap e93: 8 total, 8 up, 8 in 2024-04-07T14:39:26.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:26 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/3400244154' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/145418573"}]: dispatch 2024-04-07T14:39:26.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:26 smithi083 ceph-mon[29774]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/145418573"}]: dispatch 2024-04-07T14:39:26.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:26 smithi083 ceph-mon[29774]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:39:26.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:26 smithi189 ceph-mon[25797]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:6801/533197939"}]': finished 2024-04-07T14:39:26.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:26 smithi189 ceph-mon[25797]: osdmap e93: 8 total, 8 up, 8 in 2024-04-07T14:39:26.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:26 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/3400244154' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/145418573"}]: dispatch 2024-04-07T14:39:26.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:26 smithi189 ceph-mon[25797]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/145418573"}]: dispatch 2024-04-07T14:39:26.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:26 smithi189 ceph-mon[25797]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:39:27.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:27 smithi083 ceph-mon[24810]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/145418573"}]': finished 2024-04-07T14:39:27.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:27 smithi083 ceph-mon[24810]: osdmap e94: 8 total, 8 up, 8 in 2024-04-07T14:39:27.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:27 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/1688570478' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/1879750037"}]: dispatch 2024-04-07T14:39:27.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:27 smithi083 ceph-mon[24810]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/1879750037"}]: dispatch 2024-04-07T14:39:27.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:27 smithi083 ceph-mon[29774]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/145418573"}]': finished 2024-04-07T14:39:27.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:27 smithi083 ceph-mon[29774]: osdmap e94: 8 total, 8 up, 8 in 2024-04-07T14:39:27.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:27 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/1688570478' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/1879750037"}]: dispatch 2024-04-07T14:39:27.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:27 smithi083 ceph-mon[29774]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/1879750037"}]: dispatch 2024-04-07T14:39:27.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:27 smithi189 ceph-mon[25797]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/145418573"}]': finished 2024-04-07T14:39:27.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:27 smithi189 ceph-mon[25797]: osdmap e94: 8 total, 8 up, 8 in 2024-04-07T14:39:27.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:27 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/1688570478' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/1879750037"}]: dispatch 2024-04-07T14:39:27.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:27 smithi189 ceph-mon[25797]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/1879750037"}]: dispatch 2024-04-07T14:39:28.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:28 smithi083 ceph-mon[24810]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/1879750037"}]': finished 2024-04-07T14:39:28.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:28 smithi083 ceph-mon[24810]: osdmap e95: 8 total, 8 up, 8 in 2024-04-07T14:39:28.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:28 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/2137094980' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/1319892688"}]: dispatch 2024-04-07T14:39:28.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:28 smithi083 ceph-mon[24810]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/1319892688"}]: dispatch 2024-04-07T14:39:28.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:28 smithi083 ceph-mon[24810]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:39:28.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:28 smithi083 ceph-mon[29774]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/1879750037"}]': finished 2024-04-07T14:39:28.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:28 smithi083 ceph-mon[29774]: osdmap e95: 8 total, 8 up, 8 in 2024-04-07T14:39:28.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:28 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/2137094980' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/1319892688"}]: dispatch 2024-04-07T14:39:28.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:28 smithi083 ceph-mon[29774]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/1319892688"}]: dispatch 2024-04-07T14:39:28.677 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:28 smithi083 ceph-mon[29774]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:39:28.708 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:39:28 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:39:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:39:28.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:28 smithi189 ceph-mon[25797]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/1879750037"}]': finished 2024-04-07T14:39:28.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:28 smithi189 ceph-mon[25797]: osdmap e95: 8 total, 8 up, 8 in 2024-04-07T14:39:28.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:28 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/2137094980' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/1319892688"}]: dispatch 2024-04-07T14:39:28.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:28 smithi189 ceph-mon[25797]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/1319892688"}]: dispatch 2024-04-07T14:39:28.709 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:28 smithi189 ceph-mon[25797]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:39:29.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:29 smithi083 ceph-mon[24810]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/1319892688"}]': finished 2024-04-07T14:39:29.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:29 smithi083 ceph-mon[24810]: osdmap e96: 8 total, 8 up, 8 in 2024-04-07T14:39:29.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:29 smithi083 ceph-mon[29774]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/1319892688"}]': finished 2024-04-07T14:39:29.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:29 smithi083 ceph-mon[29774]: osdmap e96: 8 total, 8 up, 8 in 2024-04-07T14:39:29.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:29 smithi189 ceph-mon[25797]: from='client.? ' entity='client.iscsi.foo.smithi083.qmxyme' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.83:0/1319892688"}]': finished 2024-04-07T14:39:29.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:29 smithi189 ceph-mon[25797]: osdmap e96: 8 total, 8 up, 8 in 2024-04-07T14:39:30.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:30 smithi083 ceph-mon[24810]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:39:30.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:30 smithi083 ceph-mon[29774]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:39:30.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:30 smithi189 ceph-mon[25797]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:39:32.157 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:39:31 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:39:31] "GET /metrics HTTP/1.1" 200 208138 "" "Prometheus/2.33.4" 2024-04-07T14:39:33.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:32 smithi083 ceph-mon[29774]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:39:33.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:32 smithi083 ceph-mon[24810]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:39:33.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:32 smithi189 ceph-mon[25797]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:39:33.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:33 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:39:33.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:33 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:39:33.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:33 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:39:35.046 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:34 smithi083 ceph-mon[24810]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 694 B/s rd, 0 op/s 2024-04-07T14:39:35.046 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:34 smithi083 ceph-mon[29774]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 694 B/s rd, 0 op/s 2024-04-07T14:39:35.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:34 smithi189 ceph-mon[25797]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 694 B/s rd, 0 op/s 2024-04-07T14:39:35.675 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:39:35 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:39:35] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:39:36.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:36 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:36.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:36 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:36.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:36 smithi083 ceph-mon[24810]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:39:36.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:36 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:36.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:36 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:36.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:36 smithi083 ceph-mon[29774]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:39:36.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:36 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:36.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:36 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:36.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:36 smithi189 ceph-mon[25797]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:39:38.708 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:39:38 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:39:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:39:39.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:38 smithi083 ceph-mon[24810]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-07T14:39:39.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:38 smithi083 ceph-mon[29774]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-07T14:39:39.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:38 smithi189 ceph-mon[25797]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-07T14:39:41.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:40 smithi083 ceph-mon[24810]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 901 B/s rd, 0 op/s 2024-04-07T14:39:41.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:40 smithi083 ceph-mon[29774]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 901 B/s rd, 0 op/s 2024-04-07T14:39:41.176 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:39:41 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:39:41] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:39:41.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:40 smithi189 ceph-mon[25797]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 901 B/s rd, 0 op/s 2024-04-07T14:39:42.013 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:39:41 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:39:41] "GET /metrics HTTP/1.1" 200 208174 "" "Prometheus/2.33.4" 2024-04-07T14:39:42.410 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:42 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:42.410 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:42 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:42.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:42 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:42.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:42 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:42.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:42 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:42.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:42 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:43.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:43 smithi083 ceph-mon[24810]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:39:43.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:43 smithi083 ceph-mon[29774]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:39:43.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:43 smithi189 ceph-mon[25797]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:39:44.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:44 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:39:44.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:44 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:39:44.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:44 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:39:45.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:45 smithi189 ceph-mon[25797]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:39:45.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:45 smithi083 ceph-mon[24810]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:39:45.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:45 smithi083 ceph-mon[29774]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:39:46.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:46 smithi083 ceph-mon[24810]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:39:46.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:46 smithi083 ceph-mon[29774]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:39:46.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:46 smithi189 ceph-mon[25797]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:39:48.708 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:39:48 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:39:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:39:49.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:48 smithi083 ceph-mon[24810]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:39:49.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:48 smithi083 ceph-mon[29774]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:39:49.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:48 smithi189 ceph-mon[25797]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:39:50.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:39:50.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:39:50.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:39:50.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:39:50.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:39:50.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:39:51.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:50 smithi083 ceph-mon[24810]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:39:51.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:50 smithi083 ceph-mon[29774]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:39:51.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:50 smithi189 ceph-mon[25797]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:39:52.175 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:39:51 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:39:51] "GET /metrics HTTP/1.1" 200 208128 "" "Prometheus/2.33.4" 2024-04-07T14:39:53.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:52 smithi083 ceph-mon[24810]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:39:53.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:52 smithi083 ceph-mon[29774]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:39:53.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:52 smithi189 ceph-mon[25797]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:39:53.801 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:39:53 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:39:53.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:39:53.801 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:39:53 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:39:53.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.189 because it doesn't contain any IP SANs" 2024-04-07T14:39:54.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:53 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:39:54.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:53 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:39:54.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:53 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:39:55.097 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:54 smithi189 ceph-mon[25797]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:39:55.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:54 smithi083 ceph-mon[24810]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:39:55.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:54 smithi083 ceph-mon[29774]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:39:55.675 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:39:55 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:39:55] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:39:56.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:56 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:56.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:56 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:56.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:56 smithi083 ceph-mon[24810]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:39:56.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:56 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:56.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:56 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:56.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:56 smithi083 ceph-mon[29774]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:39:56.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:56 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:56.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:56 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:39:56.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:56 smithi189 ceph-mon[25797]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:39:58.708 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:39:58 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:39:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:39:59.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:39:58 smithi083 ceph-mon[24810]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:39:59.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:39:58 smithi083 ceph-mon[29774]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:39:59.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:39:58 smithi189 ceph-mon[25797]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:01.136 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:00 smithi083 ceph-mon[24810]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:01.137 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:00 smithi083 ceph-mon[24810]: overall HEALTH_OK 2024-04-07T14:40:01.137 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:00 smithi083 ceph-mon[29774]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:01.137 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:00 smithi083 ceph-mon[29774]: overall HEALTH_OK 2024-04-07T14:40:01.138 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:00 smithi189 ceph-mon[25797]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:01.138 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:00 smithi189 ceph-mon[25797]: overall HEALTH_OK 2024-04-07T14:40:01.425 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:40:01 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:40:01] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:40:02.128 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:40:01 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:40:01] "GET /metrics HTTP/1.1" 200 208128 "" "Prometheus/2.33.4" 2024-04-07T14:40:02.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:02 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:40:02.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:02 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:40:02.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:02 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:40:02.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:02 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:40:02.452 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:02 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:40:02.452 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:02 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:40:03.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:03 smithi083 ceph-mon[24810]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:40:03.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:03 smithi083 ceph-mon[29774]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:40:03.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:03 smithi189 ceph-mon[25797]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:40:03.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:40:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:40:03.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.189 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:40:03.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:40:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:40:03.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:40:03.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:40:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:40:03.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.189 because it doesn't contain any IP SANs" 2024-04-07T14:40:04.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:04 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:40:04.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:04 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:40:04.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:04 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:40:05.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:05 smithi083 ceph-mon[24810]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:05.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:05 smithi083 ceph-mon[29774]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:05.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:05 smithi189 ceph-mon[25797]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:06.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:06 smithi083 ceph-mon[24810]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:40:06.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:06 smithi083 ceph-mon[29774]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:40:06.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:06 smithi189 ceph-mon[25797]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:40:08.708 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:40:08 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:40:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:40:09.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:08 smithi083 ceph-mon[24810]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:09.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:08 smithi083 ceph-mon[29774]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:09.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:08 smithi189 ceph-mon[25797]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:11.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:10 smithi083 ceph-mon[24810]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:11.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:10 smithi083 ceph-mon[29774]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:11.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:10 smithi189 ceph-mon[25797]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:11.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:40:11 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:40:11] "GET /metrics HTTP/1.1" 200 208076 "" "Prometheus/2.33.4" 2024-04-07T14:40:12.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:12 smithi083 ceph-mon[24810]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:40:12.925 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:12 smithi083 ceph-mon[29774]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:40:13.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:12 smithi189 ceph-mon[25797]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:40:13.775 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:40:13 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:40:13.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.189 because it doesn't contain any IP SANs" 2024-04-07T14:40:13.775 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:40:13 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:40:13.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:40:13.775 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:40:13 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:40:13.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.189 because it doesn't contain any IP SANs" 2024-04-07T14:40:14.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:13 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:40:14.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:13 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:40:14.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:13 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:40:14.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:14 smithi083 ceph-mon[24810]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:14.925 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:14 smithi083 ceph-mon[29774]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:15.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:14 smithi189 ceph-mon[25797]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:15.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:40:15 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:40:15] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:40:16.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:16 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:40:16.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:16 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:40:16.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:16 smithi083 ceph-mon[24810]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:40:16.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:16 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:40:16.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:16 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:40:16.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:16 smithi083 ceph-mon[29774]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:40:16.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:16 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:40:16.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:16 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:40:16.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:16 smithi189 ceph-mon[25797]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:40:18.708 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:40:18 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:40:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:40:19.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:18 smithi083 ceph-mon[24810]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:19.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:18 smithi083 ceph-mon[29774]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:19.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:18 smithi189 ceph-mon[25797]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:20.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:19 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:40:20.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:19 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:40:20.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:19 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:40:20.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:19 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:40:20.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:19 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:40:20.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:19 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:40:20.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:19 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:40:20.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:19 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:40:20.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:19 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:40:21.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:21 smithi083 ceph-mon[24810]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:21.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:21 smithi083 ceph-mon[29774]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:21.426 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:40:21 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:40:21] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:40:21.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:21 smithi189 ceph-mon[25797]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:22.175 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:40:21 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:40:21] "GET /metrics HTTP/1.1" 200 208085 "" "Prometheus/2.33.4" 2024-04-07T14:40:22.524 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:22 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:40:22.525 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:22 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:40:22.525 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:22 smithi189 ceph-mon[25797]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:40:22.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:22 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:40:22.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:22 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:40:22.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:22 smithi083 ceph-mon[24810]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:40:22.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:22 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:40:22.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:22 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:40:22.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:22 smithi083 ceph-mon[29774]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:40:23.507 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:23 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:40:23.507 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:23 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:40:23.574 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:23 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:40:23.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:40:23 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:40:23.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.189 because it doesn't contain any IP SANs" 2024-04-07T14:40:23.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:40:23 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:40:23.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.189 because it doesn't contain any IP SANs" 2024-04-07T14:40:23.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:40:23 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:40:23.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:40:24.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:24 smithi083 ceph-mon[24810]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:24.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:24 smithi083 ceph-mon[29774]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:24.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:24 smithi189 ceph-mon[25797]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:26.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:26 smithi083 ceph-mon[24810]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:40:26.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:26 smithi083 ceph-mon[29774]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:40:26.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:26 smithi189 ceph-mon[25797]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:40:28.708 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:40:28 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:40:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:40:29.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:28 smithi083 ceph-mon[24810]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:29.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:28 smithi083 ceph-mon[29774]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:29.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:28 smithi189 ceph-mon[25797]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:31.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:30 smithi083 ceph-mon[24810]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:31.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:30 smithi083 ceph-mon[29774]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:31.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:30 smithi189 ceph-mon[25797]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:32.175 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:40:31 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:40:31] "GET /metrics HTTP/1.1" 200 208085 "" "Prometheus/2.33.4" 2024-04-07T14:40:32.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:32 smithi083 ceph-mon[24810]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:40:32.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:32 smithi083 ceph-mon[29774]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:40:33.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:32 smithi189 ceph-mon[25797]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:40:33.809 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:40:33 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:40:33.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.189 because it doesn't contain any IP SANs" 2024-04-07T14:40:33.809 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:40:33 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:40:33.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:40:33.810 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:40:33 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:40:33.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.189 because it doesn't contain any IP SANs" 2024-04-07T14:40:34.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:33 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:40:34.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:33 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:40:34.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:33 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:40:34.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:34 smithi083 ceph-mon[24810]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:34.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:34 smithi083 ceph-mon[29774]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:35.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:34 smithi189 ceph-mon[25797]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:35.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:40:35 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:40:35] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:40:36.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:36 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:40:36.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:36 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:40:36.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:36 smithi083 ceph-mon[24810]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:40:36.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:36 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:40:36.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:36 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:40:36.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:36 smithi083 ceph-mon[29774]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:40:36.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:36 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:40:36.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:36 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:40:36.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:36 smithi189 ceph-mon[25797]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:40:38.708 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:40:38 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:40:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:40:39.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:38 smithi083 ceph-mon[24810]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:39.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:38 smithi083 ceph-mon[29774]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:39.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:38 smithi189 ceph-mon[25797]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:41.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:40 smithi083 ceph-mon[24810]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:41.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:40 smithi083 ceph-mon[29774]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:41.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:40 smithi189 ceph-mon[25797]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:41.675 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:40:41 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:40:41] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:40:42.175 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:40:41 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:40:41] "GET /metrics HTTP/1.1" 200 208080 "" "Prometheus/2.33.4" 2024-04-07T14:40:42.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:42 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:40:42.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:42 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:40:42.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:42 smithi083 ceph-mon[24810]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:40:42.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:42 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:40:42.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:42 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:40:42.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:42 smithi083 ceph-mon[29774]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:40:42.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:42 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:40:42.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:42 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:40:42.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:42 smithi189 ceph-mon[25797]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:40:43.608 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:43 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:40:43.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:43 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:40:43.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:43 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:40:43.676 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:40:43 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:40:43.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.189 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:40:43.676 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:40:43 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:40:43.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:40:43.676 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:40:43 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:40:43.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.189 because it doesn't contain any IP SANs" 2024-04-07T14:40:44.569 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:44 smithi189 ceph-mon[25797]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:44.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:44 smithi083 ceph-mon[24810]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:44.675 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:44 smithi083 ceph-mon[29774]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:46.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:46 smithi189 ceph-mon[25797]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:40:46.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:46 smithi083 ceph-mon[24810]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:40:46.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:46 smithi083 ceph-mon[29774]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:40:48.708 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:40:48 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:40:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:40:49.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:48 smithi083 ceph-mon[24810]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:49.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:48 smithi083 ceph-mon[29774]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:49.207 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:48 smithi189 ceph-mon[25797]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:50.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:40:50.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:40:50.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:40:50.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:40:50.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:40:50.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:40:51.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:50 smithi083 ceph-mon[24810]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:51.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:50 smithi083 ceph-mon[29774]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:51.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:50 smithi189 ceph-mon[25797]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:52.079 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:40:51 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:40:51] "GET /metrics HTTP/1.1" 200 208107 "" "Prometheus/2.33.4" 2024-04-07T14:40:53.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:52 smithi083 ceph-mon[24810]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:40:53.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:52 smithi083 ceph-mon[29774]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:40:53.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:52 smithi189 ceph-mon[25797]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:40:53.765 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:40:53 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:40:53.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.189 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:40:53.765 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:40:53 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:40:53.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:40:53.765 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:40:53 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:40:53.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.189 because it doesn't contain any IP SANs" 2024-04-07T14:40:54.076 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:53 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:40:54.076 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:53 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:40:54.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:53 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:40:54.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:54 smithi083 ceph-mon[24810]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:54.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:54 smithi083 ceph-mon[29774]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:55.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:54 smithi189 ceph-mon[25797]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:56.175 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:40:55 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:40:55] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:40:57.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:56 smithi083 ceph-mon[24810]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:40:57.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:56 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:40:57.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:56 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:40:57.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:56 smithi083 ceph-mon[29774]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:40:57.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:56 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:40:57.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:56 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:40:57.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:56 smithi189 ceph-mon[25797]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:40:57.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:56 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:40:57.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:56 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:40:58.708 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:40:58 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:40:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:40:59.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:40:58 smithi083 ceph-mon[24810]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:59.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:40:58 smithi083 ceph-mon[29774]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:40:59.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:40:58 smithi189 ceph-mon[25797]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:01.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:00 smithi083 ceph-mon[24810]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:01.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:00 smithi083 ceph-mon[29774]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:01.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:00 smithi189 ceph-mon[25797]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:01.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:41:01 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:41:01] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:41:01.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:41:01 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:41:01] "GET /metrics HTTP/1.1" 200 208107 "" "Prometheus/2.33.4" 2024-04-07T14:41:02.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:02 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:41:02.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:02 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:41:02.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:02 smithi083 ceph-mon[24810]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:41:02.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:02 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:41:02.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:02 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:41:02.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:02 smithi083 ceph-mon[29774]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:41:02.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:02 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:41:02.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:02 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:41:02.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:02 smithi189 ceph-mon[25797]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:41:03.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:03 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:41:03.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:03 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:41:03.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:03 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:41:03.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:41:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:41:03.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.189 because it doesn't contain any IP SANs" 2024-04-07T14:41:03.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:41:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:41:03.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:41:03.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:41:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:41:03.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.189 because it doesn't contain any IP SANs" 2024-04-07T14:41:04.568 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:04 smithi189 ceph-mon[25797]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:04.826 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:04 smithi083 ceph-mon[24810]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:04.826 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:04 smithi083 ceph-mon[29774]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:06.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:06 smithi083 ceph-mon[24810]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:41:06.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:06 smithi083 ceph-mon[29774]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:41:06.684 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:06 smithi189 ceph-mon[25797]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:41:08.708 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:41:08 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:41:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:41:09.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:08 smithi083 ceph-mon[24810]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:09.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:08 smithi083 ceph-mon[29774]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:09.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:08 smithi189 ceph-mon[25797]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:11.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:10 smithi083 ceph-mon[29774]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:11.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:10 smithi083 ceph-mon[24810]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:11.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:10 smithi189 ceph-mon[25797]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:12.175 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:41:11 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:41:11] "GET /metrics HTTP/1.1" 200 208117 "" "Prometheus/2.33.4" 2024-04-07T14:41:12.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:12 smithi083 ceph-mon[24810]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:41:12.925 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:12 smithi083 ceph-mon[29774]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:41:13.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:12 smithi189 ceph-mon[25797]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:41:13.820 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:41:13 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:41:13.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.189 because it doesn't contain any IP SANs" 2024-04-07T14:41:13.820 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:41:13 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:41:13.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:41:13.820 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:41:13 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:41:13.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.189 because it doesn't contain any IP SANs" 2024-04-07T14:41:14.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:13 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:41:14.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:13 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:41:14.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:13 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:41:14.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:14 smithi083 ceph-mon[24810]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:14.925 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:14 smithi083 ceph-mon[29774]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:15.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:14 smithi189 ceph-mon[25797]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:16.175 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:41:15 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:41:15] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:41:17.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:16 smithi083 ceph-mon[24810]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:41:17.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:16 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:41:17.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:16 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:41:17.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:16 smithi083 ceph-mon[29774]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:41:17.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:16 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:41:17.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:16 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:41:17.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:16 smithi189 ceph-mon[25797]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:41:17.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:16 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:41:17.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:16 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:41:18.708 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:41:18 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:41:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:41:19.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:18 smithi083 ceph-mon[24810]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:19.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:18 smithi083 ceph-mon[29774]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:19.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:18 smithi189 ceph-mon[25797]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:20.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:19 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:41:20.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:19 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:41:20.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:19 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:41:20.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:19 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:41:20.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:19 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:41:20.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:19 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:41:20.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:19 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:41:20.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:19 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:41:20.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:19 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:41:21.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:21 smithi083 ceph-mon[24810]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:21.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:21 smithi083 ceph-mon[29774]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:21.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:21 smithi189 ceph-mon[25797]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:21.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:41:21 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:41:21] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:41:21.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:41:21 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:41:21] "GET /metrics HTTP/1.1" 200 208096 "" "Prometheus/2.33.4" 2024-04-07T14:41:22.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:22 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:41:22.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:22 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:41:22.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:22 smithi083 ceph-mon[24810]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:41:22.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:22 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:41:22.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:22 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:41:22.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:22 smithi083 ceph-mon[29774]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:41:22.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:22 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:41:22.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:22 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:41:22.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:22 smithi189 ceph-mon[25797]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:41:23.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:23 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:41:23.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:23 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:41:23.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:41:23 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:41:23.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.189 because it doesn't contain any IP SANs" 2024-04-07T14:41:23.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:41:23 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:41:23.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:41:23.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:41:23 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:41:23.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.189 because it doesn't contain any IP SANs" 2024-04-07T14:41:23.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:23 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:41:24.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:24 smithi083 ceph-mon[24810]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:24.925 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:24 smithi083 ceph-mon[29774]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:24.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:24 smithi189 ceph-mon[25797]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:26.562 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:26 smithi189 ceph-mon[25797]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:41:26.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:26 smithi083 ceph-mon[24810]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:41:26.675 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:26 smithi083 ceph-mon[29774]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:41:28.458 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:41:28 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:41:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:41:29.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:28 smithi083 ceph-mon[24810]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:29.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:28 smithi083 ceph-mon[29774]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:29.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:28 smithi189 ceph-mon[25797]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:31.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:30 smithi083 ceph-mon[24810]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:31.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:30 smithi083 ceph-mon[29774]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:31.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:30 smithi189 ceph-mon[25797]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:32.175 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:41:31 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:41:31] "GET /metrics HTTP/1.1" 200 208096 "" "Prometheus/2.33.4" 2024-04-07T14:41:33.159 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:32 smithi083 ceph-mon[24810]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:41:33.160 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:32 smithi083 ceph-mon[29774]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:41:33.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:32 smithi189 ceph-mon[25797]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:41:33.802 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:41:33 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:41:33.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.189 because it doesn't contain any IP SANs" 2024-04-07T14:41:33.803 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:41:33 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:41:33.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:41:33.803 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:41:33 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:41:33.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.189 because it doesn't contain any IP SANs" 2024-04-07T14:41:34.055 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:33 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:41:34.055 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:33 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:41:34.177 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:33 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:41:35.059 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:34 smithi083 ceph-mon[24810]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:35.059 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:34 smithi083 ceph-mon[29774]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:35.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:34 smithi189 ceph-mon[25797]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:36.425 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:41:36 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:41:36] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:41:37.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:36 smithi083 ceph-mon[24810]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:41:37.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:36 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:41:37.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:36 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:41:37.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:36 smithi083 ceph-mon[29774]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:41:37.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:36 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:41:37.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:36 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:41:37.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:37 smithi189 ceph-mon[25797]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:41:37.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:37 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:41:37.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:37 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:41:38.708 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:41:38 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:41:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:41:39.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:39 smithi083 ceph-mon[24810]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:39.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:39 smithi083 ceph-mon[29774]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:39.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:39 smithi189 ceph-mon[25797]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:41.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:41 smithi083 ceph-mon[24810]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:41.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:41 smithi083 ceph-mon[29774]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:41.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:41 smithi189 ceph-mon[25797]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:41.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:41:41 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:41:41] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:41:41.926 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:41:41 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:41:41] "GET /metrics HTTP/1.1" 200 208042 "" "Prometheus/2.33.4" 2024-04-07T14:41:42.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:42 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:41:42.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:42 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:41:42.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:42 smithi083 ceph-mon[24810]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:41:42.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:42 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:41:42.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:42 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:41:42.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:42 smithi083 ceph-mon[29774]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:41:42.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:42 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:41:42.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:42 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:41:42.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:42 smithi189 ceph-mon[25797]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:41:43.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:43 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:41:43.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:43 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:41:43.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:41:43 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:41:43.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.189 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:41:43.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:41:43 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:41:43.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:41:43.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:41:43 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:41:43.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.189 because it doesn't contain any IP SANs" 2024-04-07T14:41:43.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:43 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:41:44.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:44 smithi083 ceph-mon[24810]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:44.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:44 smithi083 ceph-mon[29774]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:44.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:44 smithi189 ceph-mon[25797]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:46.574 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:46 smithi189 ceph-mon[25797]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:41:46.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:46 smithi083 ceph-mon[24810]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:41:46.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:46 smithi083 ceph-mon[29774]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:41:48.458 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:41:48 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:41:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:41:48.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:48 smithi189 ceph-mon[25797]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:49.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:48 smithi083 ceph-mon[24810]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:49.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:48 smithi083 ceph-mon[29774]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:50.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:41:50.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:41:50.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:41:50.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:41:50.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:41:50.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:41:51.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:50 smithi083 ceph-mon[24810]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:51.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:50 smithi083 ceph-mon[29774]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:51.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:50 smithi189 ceph-mon[25797]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:52.175 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:41:51 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:41:51] "GET /metrics HTTP/1.1" 200 208047 "" "Prometheus/2.33.4" 2024-04-07T14:41:52.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:52 smithi083 ceph-mon[24810]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:41:52.925 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:52 smithi083 ceph-mon[29774]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:41:53.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:52 smithi189 ceph-mon[25797]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:41:53.837 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:41:53 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:41:53.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.189 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:41:53.837 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:41:53 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:41:53.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:41:53.837 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:41:53 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:41:53.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.189 because it doesn't contain any IP SANs" 2024-04-07T14:41:54.111 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:53 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:41:54.112 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:53 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:41:54.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:53 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:41:55.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:54 smithi083 ceph-mon[24810]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:55.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:54 smithi083 ceph-mon[29774]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:55.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:54 smithi189 ceph-mon[25797]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:56.425 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:41:56 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:41:56] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:41:57.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:57 smithi083 ceph-mon[24810]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:41:57.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:57 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:41:57.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:57 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:41:57.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:57 smithi083 ceph-mon[29774]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:41:57.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:57 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:41:57.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:57 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:41:57.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:57 smithi189 ceph-mon[25797]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:41:57.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:57 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:41:57.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:57 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:41:58.458 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:41:58 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:41:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:41:59.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:41:59 smithi083 ceph-mon[24810]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:59.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:41:59 smithi083 ceph-mon[29774]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:41:59.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:41:59 smithi189 ceph-mon[25797]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:01.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:01 smithi083 ceph-mon[24810]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:01.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:01 smithi083 ceph-mon[29774]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:01.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:01 smithi189 ceph-mon[25797]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:02.175 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:42:01 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:42:01] "GET /metrics HTTP/1.1" 200 208047 "" "Prometheus/2.33.4" 2024-04-07T14:42:02.175 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:42:01 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:42:01] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:42:03.109 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:02 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:03.109 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:02 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:03.109 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:02 smithi189 ceph-mon[25797]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:42:03.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:02 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:03.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:02 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:03.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:02 smithi083 ceph-mon[24810]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:42:03.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:02 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:03.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:02 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:03.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:02 smithi083 ceph-mon[29774]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:42:03.768 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:42:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:42:03.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": context deadline exceeded" 2024-04-07T14:42:03.768 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:42:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:42:03.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:42:03.768 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:42:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:42:03.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.189 because it doesn't contain any IP SANs" 2024-04-07T14:42:04.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:03 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:42:04.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:03 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:42:04.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:03 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:42:05.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:04 smithi083 ceph-mon[24810]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:05.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:04 smithi083 ceph-mon[29774]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:05.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:04 smithi189 ceph-mon[25797]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:06.592 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:06 smithi189 ceph-mon[25797]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:42:06.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:06 smithi083 ceph-mon[24810]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:42:06.675 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:06 smithi083 ceph-mon[29774]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:42:08.526 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:42:08 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:42:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:42:09.134 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:08 smithi083 ceph-mon[24810]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:09.134 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:08 smithi083 ceph-mon[29774]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:09.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:08 smithi189 ceph-mon[25797]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:09.935 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -e sha1=e5c885fe95ee982ff7b45214daa61b1a6d09d57d -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-04-07T14:42:10.833 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:10 smithi083 ceph-mon[24810]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:11.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:10 smithi083 ceph-mon[29774]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:11.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:10 smithi189 ceph-mon[25797]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:11.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:42:11 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:42:11] "GET /metrics HTTP/1.1" 200 208034 "" "Prometheus/2.33.4" 2024-04-07T14:42:12.778 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -e sha1=e5c885fe95ee982ff7b45214daa61b1a6d09d57d -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-04-07T14:42:12.984 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:12 smithi083 ceph-mon[24810]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:42:12.984 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:12 smithi083 ceph-mon[29774]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:42:13.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:12 smithi189 ceph-mon[25797]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:42:13.840 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:42:13 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:42:13.517Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.189 because it doesn't contain any IP SANs" 2024-04-07T14:42:13.841 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:42:13 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:42:13.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:42:13.841 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:42:13 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:42:13.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.189 because it doesn't contain any IP SANs" 2024-04-07T14:42:14.128 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:13 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:42:14.129 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:13 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:42:14.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:13 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:42:15.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:14 smithi083 ceph-mon[24810]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:15.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:14 smithi083 ceph-mon[29774]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:15.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:14 smithi189 ceph-mon[25797]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:16.307 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:42:16 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:42:16] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:42:16.630 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:16 smithi189 ceph-mon[25797]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:42:16.631 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:16 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:16.631 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:16 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:16.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:16 smithi083 ceph-mon[24810]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:42:16.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:16 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:16.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:16 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:16.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:16 smithi083 ceph-mon[29774]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:42:16.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:16 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:16.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:16 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:17.784 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -e sha1=e5c885fe95ee982ff7b45214daa61b1a6d09d57d -- bash -c 'ceph config set global log_to_journald false --force' 2024-04-07T14:42:18.708 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:42:18 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:42:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:42:19.096 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:18 smithi083 ceph-mon[24810]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:19.096 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:18 smithi083 ceph-mon[29774]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:19.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:18 smithi189 ceph-mon[25797]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:19.924 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:19 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:42:19.924 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:19 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:42:19.924 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:19 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:19.924 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:19 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:19.925 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:19 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:42:19.925 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:19 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:42:19.925 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:19 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:19.925 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:19 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:20.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:19 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:42:20.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:19 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:42:20.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:19 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:20.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:19 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:20.769 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -e sha1=e5c885fe95ee982ff7b45214daa61b1a6d09d57d -- bash -c 'ceph orch ps' 2024-04-07T14:42:21.076 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:20 smithi083 ceph-mon[24810]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:21.076 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:20 smithi083 ceph-mon[29774]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:21.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:20 smithi189 ceph-mon[25797]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:22.076 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:42:21 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:42:21] "GET /metrics HTTP/1.1" 200 208046 "" "Prometheus/2.33.4" 2024-04-07T14:42:22.076 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:42:21 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:42:21] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:42:22.786 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-07T14:42:22.787 INFO:teuthology.orchestra.run.smithi083.stdout:agent.smithi083 smithi083 running 6s ago 9m - - 2024-04-07T14:42:22.787 INFO:teuthology.orchestra.run.smithi083.stdout:agent.smithi189 smithi189 running 0s ago 9m - - 2024-04-07T14:42:22.787 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (3m) 6s ago 4m 27.1M - ba2b418f427c 9f4fc57a8aba 2024-04-07T14:42:22.787 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi189 *:3000 running (3m) 0s ago 3m 59.2M - 8.3.5 dad864ee21e9 cea72126e7bd 2024-04-07T14:42:22.787 INFO:teuthology.orchestra.run.smithi083.stdout:iscsi.foo.smithi083.qmxyme smithi083 running (3m) 6s ago 3m 51.0M - 3.5 e1d6a67b021e 81cf4f127345 2024-04-07T14:42:22.787 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi189 *:8443 running (9m) 0s ago 9m 412M - 17.2.0 e1d6a67b021e 14d5f8aeae61 2024-04-07T14:42:22.787 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:9283 running (10m) 6s ago 10m 466M - 17.2.0 e1d6a67b021e 966a9d7367c0 2024-04-07T14:42:22.787 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (10m) 6s ago 10m 52.3M 2048M 17.2.0 e1d6a67b021e 045b78f11b8e 2024-04-07T14:42:22.787 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi189 running (9m) 0s ago 9m 43.6M 2048M 17.2.0 e1d6a67b021e 76a325b30382 2024-04-07T14:42:22.787 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9m) 6s ago 9m 40.3M 2048M 17.2.0 e1d6a67b021e 69e62d2b9b32 2024-04-07T14:42:22.787 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5m) 6s ago 5m 17.5M - 1dbe0e931976 a8c27029d23c 2024-04-07T14:42:22.787 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi189 *:9100 running (5m) 0s ago 5m 17.8M - 1dbe0e931976 f7de611efab0 2024-04-07T14:42:22.788 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8m) 6s ago 8m 51.9M 4096M 17.2.0 e1d6a67b021e 8e0c47a72b20 2024-04-07T14:42:22.788 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8m) 6s ago 8m 49.7M 4096M 17.2.0 e1d6a67b021e 47f33a1b3c49 2024-04-07T14:42:22.788 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (7m) 6s ago 7m 51.3M 4096M 17.2.0 e1d6a67b021e 0747abc2af4a 2024-04-07T14:42:22.788 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7m) 6s ago 7m 47.7M 4096M 17.2.0 e1d6a67b021e 57cbb5ed0f86 2024-04-07T14:42:22.788 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi189 running (7m) 0s ago 7m 50.8M 4096M 17.2.0 e1d6a67b021e 27e179b6db24 2024-04-07T14:42:22.788 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi189 running (6m) 0s ago 6m 46.3M 4096M 17.2.0 e1d6a67b021e 9957f4f08073 2024-04-07T14:42:22.788 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi189 running (6m) 0s ago 6m 44.6M 4096M 17.2.0 e1d6a67b021e af997398de2d 2024-04-07T14:42:22.788 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi189 running (6m) 0s ago 6m 48.3M 4096M 17.2.0 e1d6a67b021e 8a39104a15b0 2024-04-07T14:42:22.788 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi189 *:9095 running (3m) 0s ago 5m 64.9M - 514e6a882f6e 1aeb972809bd 2024-04-07T14:42:22.788 INFO:teuthology.orchestra.run.smithi083.stdout:rgw.foo.smithi083.oslioj smithi083 *:8000 running (3m) 6s ago 3m 68.8M - 17.2.0 e1d6a67b021e 7c352b07e177 2024-04-07T14:42:22.788 INFO:teuthology.orchestra.run.smithi083.stdout:rgw.foo.smithi189.taxdrv smithi189 *:8000 running (3m) 0s ago 3m 72.1M - 17.2.0 e1d6a67b021e ce3b15242448 2024-04-07T14:42:23.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:22 smithi083 ceph-mon[24810]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:42:23.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:22 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:23.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:22 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:23.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:22 smithi083 ceph-mon[29774]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:42:23.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:22 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:23.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:22 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:23.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:22 smithi189 ceph-mon[25797]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:42:23.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:22 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:23.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:22 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:23.575 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -e sha1=e5c885fe95ee982ff7b45214daa61b1a6d09d57d -- bash -c 'ceph versions' 2024-04-07T14:42:23.879 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:42:23 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:42:23.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.189 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:42:23.879 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:42:23 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:42:23.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:42:23.879 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:42:23 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:42:23.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.189 because it doesn't contain any IP SANs" 2024-04-07T14:42:24.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:23 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:42:24.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:23 smithi083 ceph-mon[24810]: from='client.14997 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:42:24.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:23 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:42:24.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:23 smithi083 ceph-mon[29774]: from='client.14997 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:42:24.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:23 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:42:24.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:23 smithi189 ceph-mon[25797]: from='client.14997 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:42:25.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:24 smithi083 ceph-mon[24810]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:25.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:24 smithi083 ceph-mon[29774]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:25.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:24 smithi189 ceph-mon[25797]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:25.993 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-04-07T14:42:25.993 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-04-07T14:42:25.993 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-07T14:42:25.993 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-04-07T14:42:25.993 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-04-07T14:42:25.993 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-07T14:42:25.993 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-04-07T14:42:25.993 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-04-07T14:42:25.993 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-07T14:42:25.993 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-04-07T14:42:25.993 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": {}, 2024-04-07T14:42:25.993 INFO:teuthology.orchestra.run.smithi083.stdout: "rgw": { 2024-04-07T14:42:25.993 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-07T14:42:25.994 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-04-07T14:42:25.994 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-04-07T14:42:25.994 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 15 2024-04-07T14:42:25.994 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-04-07T14:42:25.994 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-04-07T14:42:26.596 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:26 smithi189 ceph-mon[25797]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:42:26.596 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:26 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/412821497' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-07T14:42:26.650 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:26 smithi083 ceph-mon[24810]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:42:26.651 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:26 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/412821497' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-07T14:42:26.651 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:26 smithi083 ceph-mon[29774]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:42:26.651 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:26 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/412821497' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-07T14:42:26.674 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -e sha1=e5c885fe95ee982ff7b45214daa61b1a6d09d57d -- bash -c 'ceph -s' 2024-04-07T14:42:28.669 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:42:28 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:42:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:42:28.926 INFO:teuthology.orchestra.run.smithi083.stdout: cluster: 2024-04-07T14:42:28.926 INFO:teuthology.orchestra.run.smithi083.stdout: id: 7388281a-f4eb-11ee-b649-cb9ed24678a4 2024-04-07T14:42:28.926 INFO:teuthology.orchestra.run.smithi083.stdout: health: HEALTH_OK 2024-04-07T14:42:28.926 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-07T14:42:28.926 INFO:teuthology.orchestra.run.smithi083.stdout: services: 2024-04-07T14:42:28.927 INFO:teuthology.orchestra.run.smithi083.stdout: mon: 3 daemons, quorum a,c,b (age 9m) 2024-04-07T14:42:28.927 INFO:teuthology.orchestra.run.smithi083.stdout: mgr: y(active, since 5m), standbys: x 2024-04-07T14:42:28.927 INFO:teuthology.orchestra.run.smithi083.stdout: osd: 8 osds: 8 up (since 6m), 8 in (since 6m) 2024-04-07T14:42:28.927 INFO:teuthology.orchestra.run.smithi083.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-04-07T14:42:28.927 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-07T14:42:28.927 INFO:teuthology.orchestra.run.smithi083.stdout: data: 2024-04-07T14:42:28.927 INFO:teuthology.orchestra.run.smithi083.stdout: pools: 6 pools, 161 pgs 2024-04-07T14:42:28.927 INFO:teuthology.orchestra.run.smithi083.stdout: objects: 209 objects, 585 KiB 2024-04-07T14:42:28.927 INFO:teuthology.orchestra.run.smithi083.stdout: usage: 64 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:42:28.927 INFO:teuthology.orchestra.run.smithi083.stdout: pgs: 161 active+clean 2024-04-07T14:42:28.927 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-07T14:42:28.927 INFO:teuthology.orchestra.run.smithi083.stdout: io: 2024-04-07T14:42:28.927 INFO:teuthology.orchestra.run.smithi083.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-04-07T14:42:28.927 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-07T14:42:28.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:28 smithi189 ceph-mon[25797]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:29.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:28 smithi083 ceph-mon[24810]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:29.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:28 smithi083 ceph-mon[29774]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:29.804 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -e sha1=e5c885fe95ee982ff7b45214daa61b1a6d09d57d -- bash -c 'ceph orch ls' 2024-04-07T14:42:30.117 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:29 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/2782135158' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-07T14:42:30.117 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:29 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/2782135158' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-07T14:42:30.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:29 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/2782135158' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-07T14:42:31.128 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:30 smithi083 ceph-mon[24810]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:31.129 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:30 smithi083 ceph-mon[29774]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:31.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:30 smithi189 ceph-mon[25797]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:31.903 INFO:teuthology.orchestra.run.smithi083.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-04-07T14:42:31.903 INFO:teuthology.orchestra.run.smithi083.stdout:agent 2/2 15s ago 5m * 2024-04-07T14:42:31.903 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager ?:9093,9094 1/1 15s ago 5m smithi083=a;count:1 2024-04-07T14:42:31.903 INFO:teuthology.orchestra.run.smithi083.stdout:grafana ?:3000 1/1 10s ago 5m smithi189=a;count:1 2024-04-07T14:42:31.903 INFO:teuthology.orchestra.run.smithi083.stdout:iscsi.foo 1/1 15s ago 3m count:1 2024-04-07T14:42:31.903 INFO:teuthology.orchestra.run.smithi083.stdout:mgr 2/2 15s ago 9m smithi083=y;smithi189=x;count:2 2024-04-07T14:42:31.903 INFO:teuthology.orchestra.run.smithi083.stdout:mon 3/3 15s ago 9m smithi083:172.21.15.83=a;smithi083:[v2:172.21.15.83:3301,v1:172.21.15.83:6790]=c;smithi189:172.21.15.189=b;count:3 2024-04-07T14:42:31.903 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter ?:9100 2/2 15s ago 5m smithi083=a;smithi189=b;count:2 2024-04-07T14:42:31.903 INFO:teuthology.orchestra.run.smithi083.stdout:osd 8 15s ago - 2024-04-07T14:42:31.904 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus ?:9095 1/1 10s ago 5m smithi189=a;count:1 2024-04-07T14:42:31.904 INFO:teuthology.orchestra.run.smithi083.stdout:rgw.foo ?:8000 2/2 15s ago 3m count:2 2024-04-07T14:42:31.904 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:42:31 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:42:31] "GET /metrics HTTP/1.1" 200 208046 "" "Prometheus/2.33.4" 2024-04-07T14:42:32.683 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -e sha1=e5c885fe95ee982ff7b45214daa61b1a6d09d57d -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-07T14:42:32.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:32 smithi083 ceph-mon[24810]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:42:32.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:32 smithi083 ceph-mon[24810]: from='client.24910 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:42:32.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:32 smithi083 ceph-mon[29774]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:42:32.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:32 smithi083 ceph-mon[29774]: from='client.24910 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:42:33.012 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:32 smithi189 ceph-mon[25797]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:42:33.012 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:32 smithi189 ceph-mon[25797]: from='client.24910 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:42:33.853 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:42:33 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:42:33.519Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.189 because it doesn't contain any IP SANs" 2024-04-07T14:42:34.115 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:33 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:42:34.116 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:33 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:42:34.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:33 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:42:35.033 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:34 smithi083 ceph-mon[29774]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:35.034 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:34 smithi083 ceph-mon[24810]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:35.195 INFO:teuthology.orchestra.run.smithi083.stdout:Scheduled to redeploy mgr.x on host 'smithi189' 2024-04-07T14:42:35.207 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:34 smithi189 ceph-mon[25797]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:35.886 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:35 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/2654686772' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-07T14:42:35.886 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:35 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:35.887 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:35 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:42:35.887 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:35 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:42:35.887 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:35 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:35.887 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:35 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-07T14:42:35.887 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:35 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-07T14:42:35.887 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:35 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:42:35.887 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:35 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/2654686772' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-07T14:42:35.887 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:35 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:35.887 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:35 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:42:35.887 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:35 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:42:35.887 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:35 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:35.888 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:35 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-07T14:42:35.888 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:35 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-07T14:42:35.888 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:35 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:42:36.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:35 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/2654686772' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-07T14:42:36.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:35 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:36.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:35 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:42:36.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:35 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:42:36.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:35 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:36.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:35 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-07T14:42:36.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:35 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-07T14:42:36.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:35 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:42:36.640 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:42:36 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:42:36] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:42:36.663 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -e sha1=e5c885fe95ee982ff7b45214daa61b1a6d09d57d -- bash -c 'ceph orch ps --refresh' 2024-04-07T14:42:36.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:36 smithi083 ceph-mon[24810]: from='client.15027 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:e5c885fe95ee982ff7b45214daa61b1a6d09d57d", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:42:36.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:36 smithi083 ceph-mon[24810]: Schedule redeploy daemon mgr.x 2024-04-07T14:42:36.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:36 smithi083 ceph-mon[24810]: Deploying daemon mgr.x on smithi189 2024-04-07T14:42:36.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:36 smithi083 ceph-mon[24810]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:42:36.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:36 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:36.926 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:36 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:36.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:36 smithi083 ceph-mon[29774]: from='client.15027 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:e5c885fe95ee982ff7b45214daa61b1a6d09d57d", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:42:36.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:36 smithi083 ceph-mon[29774]: Schedule redeploy daemon mgr.x 2024-04-07T14:42:36.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:36 smithi083 ceph-mon[29774]: Deploying daemon mgr.x on smithi189 2024-04-07T14:42:36.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:36 smithi083 ceph-mon[29774]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:42:36.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:36 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:36.927 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:36 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:37.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:36 smithi189 ceph-mon[25797]: from='client.15027 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:e5c885fe95ee982ff7b45214daa61b1a6d09d57d", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:42:37.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:36 smithi189 ceph-mon[25797]: Schedule redeploy daemon mgr.x 2024-04-07T14:42:37.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:36 smithi189 ceph-mon[25797]: Deploying daemon mgr.x on smithi189 2024-04-07T14:42:37.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:36 smithi189 ceph-mon[25797]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:42:37.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:36 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:37.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:36 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:38.708 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:42:38 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:42:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:42:39.163 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:38 smithi083 ceph-mon[24810]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:39.163 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:38 smithi083 ceph-mon[29774]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:39.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:38 smithi189 ceph-mon[25797]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:39.735 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-07T14:42:39.735 INFO:teuthology.orchestra.run.smithi083.stdout:agent.smithi083 smithi083 running 3s ago 10m - - 2024-04-07T14:42:39.735 INFO:teuthology.orchestra.run.smithi083.stdout:agent.smithi189 smithi189 running 17s ago 9m - - 2024-04-07T14:42:39.735 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (4m) 3s ago 4m 27.1M - ba2b418f427c 9f4fc57a8aba 2024-04-07T14:42:39.735 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi189 *:3000 running (4m) 17s ago 4m 59.2M - 8.3.5 dad864ee21e9 cea72126e7bd 2024-04-07T14:42:39.735 INFO:teuthology.orchestra.run.smithi083.stdout:iscsi.foo.smithi083.qmxyme smithi083 running (3m) 3s ago 3m 51.0M - 3.5 e1d6a67b021e 81cf4f127345 2024-04-07T14:42:39.736 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi189 *:8443 running (9m) 17s ago 9m 412M - 17.2.0 e1d6a67b021e 14d5f8aeae61 2024-04-07T14:42:39.736 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:9283 running (11m) 3s ago 11m 466M - 17.2.0 e1d6a67b021e 966a9d7367c0 2024-04-07T14:42:39.736 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (11m) 3s ago 11m 52.3M 2048M 17.2.0 e1d6a67b021e 045b78f11b8e 2024-04-07T14:42:39.736 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi189 running (9m) 17s ago 9m 43.6M 2048M 17.2.0 e1d6a67b021e 76a325b30382 2024-04-07T14:42:39.736 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (9m) 3s ago 9m 40.3M 2048M 17.2.0 e1d6a67b021e 69e62d2b9b32 2024-04-07T14:42:39.736 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (5m) 3s ago 5m 17.5M - 1dbe0e931976 a8c27029d23c 2024-04-07T14:42:39.736 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi189 *:9100 running (5m) 17s ago 5m 17.8M - 1dbe0e931976 f7de611efab0 2024-04-07T14:42:39.736 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (8m) 3s ago 8m 51.9M 4096M 17.2.0 e1d6a67b021e 8e0c47a72b20 2024-04-07T14:42:39.736 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (8m) 3s ago 8m 49.7M 4096M 17.2.0 e1d6a67b021e 47f33a1b3c49 2024-04-07T14:42:39.736 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (8m) 3s ago 8m 51.3M 4096M 17.2.0 e1d6a67b021e 0747abc2af4a 2024-04-07T14:42:39.736 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (7m) 3s ago 7m 47.7M 4096M 17.2.0 e1d6a67b021e 57cbb5ed0f86 2024-04-07T14:42:39.736 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi189 running (7m) 17s ago 7m 50.8M 4096M 17.2.0 e1d6a67b021e 27e179b6db24 2024-04-07T14:42:39.736 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi189 running (7m) 17s ago 7m 46.3M 4096M 17.2.0 e1d6a67b021e 9957f4f08073 2024-04-07T14:42:39.736 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi189 running (6m) 17s ago 6m 44.6M 4096M 17.2.0 e1d6a67b021e af997398de2d 2024-04-07T14:42:39.736 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi189 running (6m) 17s ago 6m 48.3M 4096M 17.2.0 e1d6a67b021e 8a39104a15b0 2024-04-07T14:42:39.737 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi189 *:9095 running (4m) 17s ago 5m 64.9M - 514e6a882f6e 1aeb972809bd 2024-04-07T14:42:39.737 INFO:teuthology.orchestra.run.smithi083.stdout:rgw.foo.smithi083.oslioj smithi083 *:8000 running (3m) 3s ago 3m 68.8M - 17.2.0 e1d6a67b021e 7c352b07e177 2024-04-07T14:42:39.737 INFO:teuthology.orchestra.run.smithi083.stdout:rgw.foo.smithi189.taxdrv smithi189 *:8000 running (3m) 17s ago 3m 72.1M - 17.2.0 e1d6a67b021e ce3b15242448 2024-04-07T14:42:40.623 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -e sha1=e5c885fe95ee982ff7b45214daa61b1a6d09d57d -- bash -c 'sleep 180' 2024-04-07T14:42:41.170 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:40 smithi083 ceph-mon[24810]: from='client.24928 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:42:41.171 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:40 smithi083 ceph-mon[24810]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:41.171 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:40 smithi083 ceph-mon[29774]: from='client.24928 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:42:41.171 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:40 smithi083 ceph-mon[29774]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:41.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:40 smithi189 ceph-mon[25797]: from='client.24928 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:42:41.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:40 smithi189 ceph-mon[25797]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:41.961 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:42:41 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:42:41] "GET /metrics HTTP/1.1" 200 208038 "" "Prometheus/2.33.4" 2024-04-07T14:42:42.305 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:42:41 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:42:41] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:42:43.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:42 smithi083 ceph-mon[29774]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:42:43.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:42 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:43.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:42 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:43.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:42 smithi083 ceph-mon[24810]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:42:43.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:42 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:43.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:42 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:43.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:42 smithi189 ceph-mon[25797]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:42:43.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:42 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:43.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:42 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:44.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:43 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:42:44.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:43 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:42:44.459 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:43 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:42:45.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:44 smithi083 ceph-mon[24810]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:45.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:44 smithi083 ceph-mon[29774]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:45.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:44 smithi189 ceph-mon[25797]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:46.571 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:46 smithi189 ceph-mon[25797]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:42:46.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:46 smithi083 ceph-mon[24810]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:42:46.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:46 smithi083 ceph-mon[29774]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:42:48.708 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:42:48 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:42:48] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:42:49.179 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:48 smithi083 ceph-mon[24810]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:49.179 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:48 smithi083 ceph-mon[29774]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:49.207 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:48 smithi189 ceph-mon[25797]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:50.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:42:50.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:42:50.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:42:50.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:42:50.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:42:50.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:42:51.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:50 smithi083 ceph-mon[24810]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:51.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:50 smithi083 ceph-mon[29774]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:51.207 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:50 smithi189 ceph-mon[25797]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:52.175 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:42:51 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:42:51] "GET /metrics HTTP/1.1" 200 208039 "" "Prometheus/2.33.4" 2024-04-07T14:42:53.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:52 smithi083 ceph-mon[24810]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:42:53.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:52 smithi083 ceph-mon[29774]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:42:53.207 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:52 smithi189 ceph-mon[25797]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:42:54.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:53 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:42:54.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:53 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:42:54.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:53 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:42:54.980 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:54 smithi083 ceph-mon[24810]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:54.980 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:54 smithi083 ceph-mon[29774]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:55.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:54 smithi189 ceph-mon[25797]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:56.419 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:56 smithi083 ceph-mon[24810]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:42:56.420 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:56 smithi083 ceph-mon[29774]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:42:56.675 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:42:56 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:42:56] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:42:56.707 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:56 smithi189 ceph-mon[25797]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:42:57.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:57 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:57.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:57 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:57.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:57 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:57.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:57 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:57.707 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:57 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:57.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:57 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:42:58.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:42:58 smithi083 ceph-mon[24810]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:58.675 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:42:58 smithi083 ceph-mon[29774]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:58.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:42:58 smithi189 ceph-mon[25797]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:42:58.708 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:42:58 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[27349]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:42:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:43:01.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:00 smithi083 ceph-mon[24810]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:01.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:00 smithi083 ceph-mon[29774]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:01.207 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:00 smithi189 ceph-mon[25797]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:02.101 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:43:01 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:43:01] "GET /metrics HTTP/1.1" 200 208039 "" "Prometheus/2.33.4" 2024-04-07T14:43:02.425 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:43:02 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:43:02] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:43:03.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:03 smithi189 ceph-mon[25797]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:43:03.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:03 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:03.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:03 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:03.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:03 smithi083 ceph-mon[24810]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:43:03.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:03 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:03.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:03 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:03.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:03 smithi083 ceph-mon[29774]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:43:03.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:03 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:03.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:03 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:03.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:43:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:43:03.507Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.189 because it doesn't contain any IP SANs" 2024-04-07T14:43:03.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:43:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:43:03.508Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:43:04.338 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:04 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:43:04.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:04 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:43:04.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:04 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:43:04.608 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:04 smithi189 systemd[1]: Stopping Ceph mgr.x for 7388281a-f4eb-11ee-b649-cb9ed24678a4... 2024-04-07T14:43:04.608 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:04 smithi189 bash[50443]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr.x" found: no such container 2024-04-07T14:43:04.913 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:04 smithi189 podman[50456]: 2024-04-07 14:43:04.609929641 +0000 UTC m=+0.123840488 container died 14d5f8aeae614d501de1af6c275a1410e9ebe8715451e0cd16abcaec5ba19b95 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x, ceph=True, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, distribution-scope=public, architecture=x86_64, maintainer=Guillaume Abrioux , release=754, name=centos-stream, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, version=8, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, vcs-type=git) 2024-04-07T14:43:05.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:05 smithi083 ceph-mon[24810]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:05.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:05 smithi083 ceph-mon[29774]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:05.176 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:43:04 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:43:04.907Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=3 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:43:05.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:05 smithi189 ceph-mon[25797]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:05.208 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:05 smithi189 podman[50456]: 2024-04-07 14:43:05.022832005 +0000 UTC m=+0.536742850 container cleanup 14d5f8aeae614d501de1af6c275a1410e9ebe8715451e0cd16abcaec5ba19b95 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x, RELEASE=HEAD, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, release=754, distribution-scope=public, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, GIT_CLEAN=True, version=8, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True) 2024-04-07T14:43:05.208 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:05 smithi189 bash[50456]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x 2024-04-07T14:43:05.208 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:05 smithi189 bash[50484]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr.x" found: no such container 2024-04-07T14:43:05.708 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:05 smithi189 podman[50470]: 2024-04-07 14:43:05.299656354 +0000 UTC m=+0.681930515 container remove 14d5f8aeae614d501de1af6c275a1410e9ebe8715451e0cd16abcaec5ba19b95 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , name=centos-stream, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, io.buildah.version=1.19.8, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, ceph=True, vendor=Red Hat, Inc., RELEASE=HEAD) 2024-04-07T14:43:05.708 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:05 smithi189 systemd[1]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-07T14:43:05.708 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:05 smithi189 systemd[1]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@mgr.x.service: Failed with result 'exit-code'. 2024-04-07T14:43:05.708 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:05 smithi189 systemd[1]: Stopped Ceph mgr.x for 7388281a-f4eb-11ee-b649-cb9ed24678a4. 2024-04-07T14:43:05.708 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:05 smithi189 systemd[1]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@mgr.x.service: Consumed 17.262s CPU time. 2024-04-07T14:43:05.709 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:05 smithi189 systemd[1]: Starting Ceph mgr.x for 7388281a-f4eb-11ee-b649-cb9ed24678a4... 2024-04-07T14:43:05.967 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:05 smithi189 podman[50587]: 2024-04-07 14:43:05.615939347 +0000 UTC m=+0.018380228 image pull fe467d9065a1c6ff757668cf0f97eb2a00e56f51271fffd4b076c4d1580a91ce quay.ceph.io/ceph-ci/ceph:e5c885fe95ee982ff7b45214daa61b1a6d09d57d 2024-04-07T14:43:05.968 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:05 smithi189 podman[50587]: 2024-04-07 14:43:05.734163583 +0000 UTC m=+0.136604452 container create 0ef726e5100dc85819cb69a35bea77af315f205e88937c66865b6561d5b5f195 (image=quay.ceph.io/ceph-ci/ceph:e5c885fe95ee982ff7b45214daa61b1a6d09d57d, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=main-e5c885f, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-04-07T14:43:06.220 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:05 smithi189 podman[50587]: 2024-04-07 14:43:05.970115601 +0000 UTC m=+0.372556466 container init 0ef726e5100dc85819cb69a35bea77af315f205e88937c66865b6561d5b5f195 (image=quay.ceph.io/ceph-ci/ceph:e5c885fe95ee982ff7b45214daa61b1a6d09d57d, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x, CEPH_POINT_RELEASE=, RELEASE=main-e5c885f, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-07T14:43:06.221 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:05 smithi189 podman[50587]: 2024-04-07 14:43:05.972949946 +0000 UTC m=+0.375390815 container start 0ef726e5100dc85819cb69a35bea77af315f205e88937c66865b6561d5b5f195 (image=quay.ceph.io/ceph-ci/ceph:e5c885fe95ee982ff7b45214daa61b1a6d09d57d, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, RELEASE=main-e5c885f) 2024-04-07T14:43:06.221 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:05 smithi189 ceph-mgr[50606]: -- 172.21.15.189:0/577468945 <== mon.2 v2:172.21.15.189:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5572dac925a0 con 0x5572dac66800 2024-04-07T14:43:06.221 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.033+0000 7ff29de4f200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-07T14:43:06.221 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.033+0000 7ff29de4f200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-07T14:43:06.221 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.221 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.037+0000 7ff29de4f200 -1 mgr[py] Class not found in module 'restful' 2024-04-07T14:43:06.221 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.037+0000 7ff29de4f200 -1 mgr[py] Error loading module 'restful': (22) Invalid argument 2024-04-07T14:43:06.221 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.052+0000 7ff29de4f200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-07T14:43:06.221 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.052+0000 7ff29de4f200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-07T14:43:06.221 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.221 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.056+0000 7ff29de4f200 -1 mgr[py] Class not found in module 'mds_autoscaler' 2024-04-07T14:43:06.221 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.056+0000 7ff29de4f200 -1 mgr[py] Error loading module 'mds_autoscaler': (22) Invalid argument 2024-04-07T14:43:06.221 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.071+0000 7ff29de4f200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-07T14:43:06.222 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.071+0000 7ff29de4f200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-07T14:43:06.222 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.222 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.075+0000 7ff29de4f200 -1 mgr[py] Class not found in module 'osd_perf_query' 2024-04-07T14:43:06.222 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.075+0000 7ff29de4f200 -1 mgr[py] Error loading module 'osd_perf_query': (22) Invalid argument 2024-04-07T14:43:06.222 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 bash[50587]: 0ef726e5100dc85819cb69a35bea77af315f205e88937c66865b6561d5b5f195 2024-04-07T14:43:06.222 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 systemd[1]: Started Ceph mgr.x for 7388281a-f4eb-11ee-b649-cb9ed24678a4. 2024-04-07T14:43:06.222 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.093+0000 7ff29de4f200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-07T14:43:06.223 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.093+0000 7ff29de4f200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-07T14:43:06.223 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.223 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.099+0000 7ff29de4f200 -1 mgr[py] Class not found in module 'snap_schedule' 2024-04-07T14:43:06.223 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.099+0000 7ff29de4f200 -1 mgr[py] Error loading module 'snap_schedule': (22) Invalid argument 2024-04-07T14:43:06.223 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.115+0000 7ff29de4f200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-07T14:43:06.223 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.115+0000 7ff29de4f200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-07T14:43:06.223 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.223 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.119+0000 7ff29de4f200 -1 mgr[py] Class not found in module 'dashboard' 2024-04-07T14:43:06.223 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.119+0000 7ff29de4f200 -1 mgr[py] Error loading module 'dashboard': (22) Invalid argument 2024-04-07T14:43:06.223 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.137+0000 7ff29de4f200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-07T14:43:06.223 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.137+0000 7ff29de4f200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-07T14:43:06.223 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.223 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.142+0000 7ff29de4f200 -1 mgr[py] Class not found in module 'zabbix' 2024-04-07T14:43:06.223 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.142+0000 7ff29de4f200 -1 mgr[py] Error loading module 'zabbix': (22) Invalid argument 2024-04-07T14:43:06.224 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.157+0000 7ff29de4f200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-07T14:43:06.224 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.157+0000 7ff29de4f200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-07T14:43:06.224 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.224 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.162+0000 7ff29de4f200 -1 mgr[py] Class not found in module 'balancer' 2024-04-07T14:43:06.224 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.162+0000 7ff29de4f200 -1 mgr[py] Error loading module 'balancer': (22) Invalid argument 2024-04-07T14:43:06.224 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.178+0000 7ff29de4f200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-07T14:43:06.224 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.178+0000 7ff29de4f200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-07T14:43:06.224 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.224 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.183+0000 7ff29de4f200 -1 mgr[py] Class not found in module 'influx' 2024-04-07T14:43:06.224 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.183+0000 7ff29de4f200 -1 mgr[py] Error loading module 'influx': (22) Invalid argument 2024-04-07T14:43:06.224 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.213+0000 7ff29de4f200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-07T14:43:06.224 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.213+0000 7ff29de4f200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-07T14:43:06.224 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.224 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.217+0000 7ff29de4f200 -1 mgr[py] Class not found in module 'alerts' 2024-04-07T14:43:06.224 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.217+0000 7ff29de4f200 -1 mgr[py] Error loading module 'alerts': (22) Invalid argument 2024-04-07T14:43:06.471 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.233+0000 7ff29de4f200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-07T14:43:06.471 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.233+0000 7ff29de4f200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-07T14:43:06.471 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.471 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.238+0000 7ff29de4f200 -1 mgr[py] Class not found in module 'iostat' 2024-04-07T14:43:06.471 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.238+0000 7ff29de4f200 -1 mgr[py] Error loading module 'iostat': (22) Invalid argument 2024-04-07T14:43:06.471 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.253+0000 7ff29de4f200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-07T14:43:06.471 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.253+0000 7ff29de4f200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-07T14:43:06.472 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.472 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.257+0000 7ff29de4f200 -1 mgr[py] Class not found in module 'rgw' 2024-04-07T14:43:06.472 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.257+0000 7ff29de4f200 -1 mgr[py] Error loading module 'rgw': (22) Invalid argument 2024-04-07T14:43:06.472 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.272+0000 7ff29de4f200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-07T14:43:06.472 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.272+0000 7ff29de4f200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-07T14:43:06.472 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.472 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.276+0000 7ff29de4f200 -1 mgr[py] Class not found in module 'rbd_support' 2024-04-07T14:43:06.472 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.276+0000 7ff29de4f200 -1 mgr[py] Error loading module 'rbd_support': (22) Invalid argument 2024-04-07T14:43:06.472 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.292+0000 7ff29de4f200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-07T14:43:06.472 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.292+0000 7ff29de4f200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-07T14:43:06.472 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.472 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.296+0000 7ff29de4f200 -1 mgr[py] Class not found in module 'progress' 2024-04-07T14:43:06.472 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.296+0000 7ff29de4f200 -1 mgr[py] Error loading module 'progress': (22) Invalid argument 2024-04-07T14:43:06.472 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.311+0000 7ff29de4f200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-07T14:43:06.473 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.311+0000 7ff29de4f200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-07T14:43:06.473 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.473 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.315+0000 7ff29de4f200 -1 mgr[py] Class not found in module 'pg_autoscaler' 2024-04-07T14:43:06.473 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.315+0000 7ff29de4f200 -1 mgr[py] Error loading module 'pg_autoscaler': (22) Invalid argument 2024-04-07T14:43:06.473 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.331+0000 7ff29de4f200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-07T14:43:06.473 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.331+0000 7ff29de4f200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-07T14:43:06.473 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.474 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.335+0000 7ff29de4f200 -1 mgr[py] Class not found in module 'devicehealth' 2024-04-07T14:43:06.474 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.335+0000 7ff29de4f200 -1 mgr[py] Error loading module 'devicehealth': (22) Invalid argument 2024-04-07T14:43:06.474 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.350+0000 7ff29de4f200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-07T14:43:06.474 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.350+0000 7ff29de4f200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-07T14:43:06.474 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.474 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.354+0000 7ff29de4f200 -1 mgr[py] Class not found in module 'k8sevents' 2024-04-07T14:43:06.474 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.354+0000 7ff29de4f200 -1 mgr[py] Error loading module 'k8sevents': (22) Invalid argument 2024-04-07T14:43:06.474 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.370+0000 7ff29de4f200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-07T14:43:06.474 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.370+0000 7ff29de4f200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-07T14:43:06.474 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.474 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.374+0000 7ff29de4f200 -1 mgr[py] Class not found in module 'rook' 2024-04-07T14:43:06.474 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.374+0000 7ff29de4f200 -1 mgr[py] Error loading module 'rook': (22) Invalid argument 2024-04-07T14:43:06.474 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.392+0000 7ff29de4f200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-07T14:43:06.474 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.392+0000 7ff29de4f200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-07T14:43:06.475 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.475 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.396+0000 7ff29de4f200 -1 mgr[py] Class not found in module 'diskprediction_local' 2024-04-07T14:43:06.475 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.396+0000 7ff29de4f200 -1 mgr[py] Error loading module 'diskprediction_local': (22) Invalid argument 2024-04-07T14:43:06.475 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.434+0000 7ff29de4f200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-07T14:43:06.475 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.434+0000 7ff29de4f200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-07T14:43:06.475 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.475 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.438+0000 7ff29de4f200 -1 mgr[py] Class not found in module 'mirroring' 2024-04-07T14:43:06.475 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.438+0000 7ff29de4f200 -1 mgr[py] Error loading module 'mirroring': (22) Invalid argument 2024-04-07T14:43:06.475 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.452+0000 7ff29de4f200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-07T14:43:06.475 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.452+0000 7ff29de4f200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-07T14:43:06.475 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.475 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.456+0000 7ff29de4f200 -1 mgr[py] Class not found in module 'selftest' 2024-04-07T14:43:06.475 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.456+0000 7ff29de4f200 -1 mgr[py] Error loading module 'selftest': (22) Invalid argument 2024-04-07T14:43:06.475 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.471+0000 7ff29de4f200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-07T14:43:06.730 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.471+0000 7ff29de4f200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-07T14:43:06.731 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.731 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.476+0000 7ff29de4f200 -1 mgr[py] Class not found in module 'telegraf' 2024-04-07T14:43:06.731 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.476+0000 7ff29de4f200 -1 mgr[py] Error loading module 'telegraf': (22) Invalid argument 2024-04-07T14:43:06.731 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.491+0000 7ff29de4f200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-07T14:43:06.731 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.491+0000 7ff29de4f200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-07T14:43:06.731 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.731 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.495+0000 7ff29de4f200 -1 mgr[py] Class not found in module 'test_orchestrator' 2024-04-07T14:43:06.731 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.495+0000 7ff29de4f200 -1 mgr[py] Error loading module 'test_orchestrator': (22) Invalid argument 2024-04-07T14:43:06.731 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.510+0000 7ff29de4f200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-07T14:43:06.731 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.510+0000 7ff29de4f200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-07T14:43:06.731 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.731 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.515+0000 7ff29de4f200 -1 mgr[py] Class not found in module 'localpool' 2024-04-07T14:43:06.732 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.515+0000 7ff29de4f200 -1 mgr[py] Error loading module 'localpool': (22) Invalid argument 2024-04-07T14:43:06.732 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.533+0000 7ff29de4f200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-07T14:43:06.732 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.533+0000 7ff29de4f200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-07T14:43:06.732 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.732 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.538+0000 7ff29de4f200 -1 mgr[py] Class not found in module 'crash' 2024-04-07T14:43:06.732 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.538+0000 7ff29de4f200 -1 mgr[py] Error loading module 'crash': (22) Invalid argument 2024-04-07T14:43:06.732 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.556+0000 7ff29de4f200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-07T14:43:06.732 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.556+0000 7ff29de4f200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-07T14:43:06.732 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.732 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.561+0000 7ff29de4f200 -1 mgr[py] Class not found in module 'insights' 2024-04-07T14:43:06.732 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.561+0000 7ff29de4f200 -1 mgr[py] Error loading module 'insights': (22) Invalid argument 2024-04-07T14:43:06.732 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.577+0000 7ff29de4f200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-07T14:43:06.732 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.577+0000 7ff29de4f200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-07T14:43:06.733 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.733 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.582+0000 7ff29de4f200 -1 mgr[py] Class not found in module 'orchestrator' 2024-04-07T14:43:06.733 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.582+0000 7ff29de4f200 -1 mgr[py] Error loading module 'orchestrator': (22) Invalid argument 2024-04-07T14:43:06.733 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.598+0000 7ff29de4f200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-07T14:43:06.733 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.598+0000 7ff29de4f200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-07T14:43:06.733 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.733 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.602+0000 7ff29de4f200 -1 mgr[py] Class not found in module 'osd_support' 2024-04-07T14:43:06.733 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.602+0000 7ff29de4f200 -1 mgr[py] Error loading module 'osd_support': (22) Invalid argument 2024-04-07T14:43:06.733 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.617+0000 7ff29de4f200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-07T14:43:06.733 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.617+0000 7ff29de4f200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-07T14:43:06.733 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.733 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.621+0000 7ff29de4f200 -1 mgr[py] Class not found in module 'volumes' 2024-04-07T14:43:06.733 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.621+0000 7ff29de4f200 -1 mgr[py] Error loading module 'volumes': (22) Invalid argument 2024-04-07T14:43:06.733 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.673+0000 7ff29de4f200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-07T14:43:06.733 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.673+0000 7ff29de4f200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-07T14:43:06.734 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.734 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.677+0000 7ff29de4f200 -1 mgr[py] Class not found in module 'stats' 2024-04-07T14:43:06.734 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.677+0000 7ff29de4f200 -1 mgr[py] Error loading module 'stats': (22) Invalid argument 2024-04-07T14:43:06.734 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.693+0000 7ff29de4f200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-07T14:43:06.734 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.693+0000 7ff29de4f200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-07T14:43:06.734 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.734 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.697+0000 7ff29de4f200 -1 mgr[py] Class not found in module 'cephadm' 2024-04-07T14:43:06.734 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.697+0000 7ff29de4f200 -1 mgr[py] Error loading module 'cephadm': (22) Invalid argument 2024-04-07T14:43:06.734 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.712+0000 7ff29de4f200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-07T14:43:06.734 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.712+0000 7ff29de4f200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-07T14:43:06.734 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.734 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.716+0000 7ff29de4f200 -1 mgr[py] Class not found in module 'telemetry' 2024-04-07T14:43:06.735 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.716+0000 7ff29de4f200 -1 mgr[py] Error loading module 'telemetry': (22) Invalid argument 2024-04-07T14:43:06.735 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.731+0000 7ff29de4f200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-07T14:43:07.129 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.731+0000 7ff29de4f200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-07T14:43:07.129 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:07.129 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.735+0000 7ff29de4f200 -1 mgr[py] Class not found in module 'prometheus' 2024-04-07T14:43:07.129 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.735+0000 7ff29de4f200 -1 mgr[py] Error loading module 'prometheus': (22) Invalid argument 2024-04-07T14:43:07.129 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.751+0000 7ff29de4f200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-07T14:43:07.129 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.751+0000 7ff29de4f200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-07T14:43:07.129 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:07.129 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.755+0000 7ff29de4f200 -1 mgr[py] Class not found in module 'status' 2024-04-07T14:43:07.129 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.755+0000 7ff29de4f200 -1 mgr[py] Error loading module 'status': (22) Invalid argument 2024-04-07T14:43:07.129 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.770+0000 7ff29de4f200 -1 mgr[py] Module not found: 'mgr_module' 2024-04-07T14:43:07.129 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.770+0000 7ff29de4f200 -1 mgr[py] ModuleNotFoundError: No module named 'mgr_module' 2024-04-07T14:43:07.130 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:07.130 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.774+0000 7ff29de4f200 -1 mgr[py] Class not found in module 'nfs' 2024-04-07T14:43:07.130 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.774+0000 7ff29de4f200 -1 mgr[py] Error loading module 'nfs': (22) Invalid argument 2024-04-07T14:43:07.130 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:43:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:43:06.774+0000 7ff29de4f200 -1 log_channel(cluster) log [ERR] : Failed to load ceph-mgr modules: restful, mds_autoscaler, osd_perf_query, snap_schedule, dashboard, zabbix, balancer, influx, alerts, iostat, rgw, rbd_support, progress, pg_autoscaler, devicehealth, k8sevents, rook, diskprediction_local, mirroring, selftest, telegraf, test_orchestrator, localpool, crash, insights, orchestrator, osd_support, volumes, stats, cephadm, telemetry, prometheus, status, nfs 2024-04-07T14:43:07.382 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:07 smithi189 ceph-mon[25797]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:43:07.382 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:07 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:07.382 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:07 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:07.383 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:07 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:43:07.383 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:07 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:43:07.383 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:07 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:07.383 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:07 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:07.383 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:07 smithi189 ceph-mon[25797]: Standby manager daemon x restarted 2024-04-07T14:43:07.383 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:07 smithi189 ceph-mon[25797]: Standby manager daemon x started 2024-04-07T14:43:07.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:07 smithi083 ceph-mon[29774]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:43:07.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:07 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:07.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:07 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:07.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:07 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:43:07.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:07 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:43:07.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:07 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:07.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:07 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:07.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:07 smithi083 ceph-mon[29774]: Standby manager daemon x restarted 2024-04-07T14:43:07.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:07 smithi083 ceph-mon[29774]: Standby manager daemon x started 2024-04-07T14:43:07.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:07 smithi083 ceph-mon[24810]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:43:07.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:07 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:07.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:07 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:07.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:07 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:43:07.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:07 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:43:07.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:07 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:07.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:07 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:07.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:07 smithi083 ceph-mon[24810]: Standby manager daemon x restarted 2024-04-07T14:43:07.427 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:07 smithi083 ceph-mon[24810]: Standby manager daemon x started 2024-04-07T14:43:08.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:08 smithi083 ceph-mon[24810]: mgrmap e21: y(active, since 6m), standbys: x 2024-04-07T14:43:08.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:08 smithi083 ceph-mon[24810]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:08.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:08 smithi083 ceph-mon[29774]: mgrmap e21: y(active, since 6m), standbys: x 2024-04-07T14:43:08.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:08 smithi083 ceph-mon[29774]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:08.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:08 smithi189 ceph-mon[25797]: mgrmap e21: y(active, since 6m), standbys: x 2024-04-07T14:43:08.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:08 smithi189 ceph-mon[25797]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:09.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:09 smithi083 ceph-mon[24810]: Failed to load ceph-mgr modules: restful, mds_autoscaler, osd_perf_query, snap_schedule, dashboard, zabbix, balancer, influx, alerts, iostat, rgw, rbd_support, progress, pg_autoscaler, devicehealth, k8sevents, rook, diskprediction_local, mirroring, selftest, telegraf, test_orchestrator, localpool, crash, insights, orchestrator, osd_support, volumes, stats, cephadm, telemetry, prometheus, status, nfs 2024-04-07T14:43:09.925 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:09 smithi083 ceph-mon[29774]: Failed to load ceph-mgr modules: restful, mds_autoscaler, osd_perf_query, snap_schedule, dashboard, zabbix, balancer, influx, alerts, iostat, rgw, rbd_support, progress, pg_autoscaler, devicehealth, k8sevents, rook, diskprediction_local, mirroring, selftest, telegraf, test_orchestrator, localpool, crash, insights, orchestrator, osd_support, volumes, stats, cephadm, telemetry, prometheus, status, nfs 2024-04-07T14:43:09.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:09 smithi189 ceph-mon[25797]: Failed to load ceph-mgr modules: restful, mds_autoscaler, osd_perf_query, snap_schedule, dashboard, zabbix, balancer, influx, alerts, iostat, rgw, rbd_support, progress, pg_autoscaler, devicehealth, k8sevents, rook, diskprediction_local, mirroring, selftest, telegraf, test_orchestrator, localpool, crash, insights, orchestrator, osd_support, volumes, stats, cephadm, telemetry, prometheus, status, nfs 2024-04-07T14:43:10.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:10 smithi083 ceph-mon[24810]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:10.925 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:10 smithi083 ceph-mon[29774]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:10.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:10 smithi189 ceph-mon[25797]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:12.175 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:43:11 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:43:11] "GET /metrics HTTP/1.1" 200 208034 "" "Prometheus/2.33.4" 2024-04-07T14:43:13.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:12 smithi083 ceph-mon[24810]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:43:13.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:12 smithi083 ceph-mon[29774]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:43:13.207 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:12 smithi189 ceph-mon[25797]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:43:13.813 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:43:13 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:43:13.506Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:43:13.814 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:43:13 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:43:13.507Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:43:13.814 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:43:13 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:43:13.508Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:43:14.087 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:13 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:43:14.087 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:13 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:43:14.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:13 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:43:15.125 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:14 smithi083 ceph-mon[24810]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:15.126 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:14 smithi083 ceph-mon[29774]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:15.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:14 smithi189 ceph-mon[25797]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:16.399 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:16 smithi083 ceph-mon[24810]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:43:16.400 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:16 smithi083 ceph-mon[29774]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:43:16.675 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:43:16 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:43:16] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:43:16.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:16 smithi189 ceph-mon[25797]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:43:17.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:17 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:17.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:17 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:17.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:17 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:17.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:17 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:17.925 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:17 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:17.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:17 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:18.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:18 smithi083 ceph-mon[24810]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:18.925 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:18 smithi083 ceph-mon[29774]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:18.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:18 smithi189 ceph-mon[25797]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:21.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:20 smithi083 ceph-mon[24810]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:21.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:20 smithi083 ceph-mon[29774]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:21.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:20 smithi189 ceph-mon[25797]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:22.175 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:43:21 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:43:21] "GET /metrics HTTP/1.1" 200 208042 "" "Prometheus/2.33.4" 2024-04-07T14:43:22.457 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:43:22 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:43:22] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:43:23.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:22 smithi083 ceph-mon[29774]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:43:23.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:22 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:23.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:22 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:23.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:22 smithi083 ceph-mon[24810]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:43:23.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:22 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:23.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:22 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:23.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:22 smithi189 ceph-mon[25797]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:43:23.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:22 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:23.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:22 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:23.856 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:43:23 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:43:23.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:43:23.857 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:43:23 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:43:23.508Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:43:23.857 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:43:23 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:43:23.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:43:23.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:23 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:43:24.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:23 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:43:24.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:23 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:43:25.161 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:24 smithi189 ceph-mon[25797]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:25.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:24 smithi083 ceph-mon[24810]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:25.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:24 smithi083 ceph-mon[29774]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:26.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:26 smithi083 ceph-mon[24810]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:43:26.675 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:43:26 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:43:26] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:43:26.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:26 smithi083 ceph-mon[29774]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:43:26.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:26 smithi189 ceph-mon[25797]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:43:27.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:27 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:27.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:27 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:27.675 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:27 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:27.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:27 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:27.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:27 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:27.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:27 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:28.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:28 smithi083 ceph-mon[24810]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:28.675 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:28 smithi083 ceph-mon[29774]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:28.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:28 smithi189 ceph-mon[25797]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:31.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:30 smithi083 ceph-mon[24810]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:31.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:30 smithi083 ceph-mon[29774]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:31.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:30 smithi189 ceph-mon[25797]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:32.175 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:43:31 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:43:31] "GET /metrics HTTP/1.1" 200 208042 "" "Prometheus/2.33.4" 2024-04-07T14:43:33.147 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:32 smithi189 ceph-mon[25797]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:43:33.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:32 smithi083 ceph-mon[24810]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:43:33.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:32 smithi083 ceph-mon[29774]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:43:33.859 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:43:33 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:43:33.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:43:33.860 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:43:33 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:43:33.508Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:43:33.860 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:43:33 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:43:33.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:43:34.169 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:33 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:43:34.170 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:33 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:43:34.207 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:33 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:43:35.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:34 smithi083 ceph-mon[24810]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:35.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:34 smithi083 ceph-mon[29774]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:35.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:34 smithi189 ceph-mon[25797]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:36.627 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:36 smithi083 ceph-mon[24810]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:43:36.627 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:36 smithi083 ceph-mon[29774]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:43:36.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:36 smithi189 ceph-mon[25797]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:43:36.914 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:43:36 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:43:36] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:43:37.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:37 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:37.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:37 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:37.925 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:37 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:37.926 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:37 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:37.957 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:37 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:37.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:37 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:38.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:38 smithi083 ceph-mon[24810]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:38.925 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:38 smithi083 ceph-mon[29774]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:38.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:38 smithi189 ceph-mon[25797]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:41.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:40 smithi083 ceph-mon[24810]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:41.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:40 smithi083 ceph-mon[29774]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:41.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:40 smithi189 ceph-mon[25797]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:42.175 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:43:41 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:43:41] "GET /metrics HTTP/1.1" 200 208041 "" "Prometheus/2.33.4" 2024-04-07T14:43:43.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:42 smithi083 ceph-mon[24810]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:43:43.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:42 smithi083 ceph-mon[29774]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:43:43.207 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:42 smithi189 ceph-mon[25797]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:43:43.860 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:43:43 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:43:43] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:43:43.861 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:43:43 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:43:43.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:43:43.861 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:43:43 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:43:43.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:43:43.861 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:43:43 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:43:43.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:43:44.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:43 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:43:44.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:43 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:44.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:43 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:44.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:43 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:43:44.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:43 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:44.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:43 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:44.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:43 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:43:44.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:43 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:44.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:43 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:45.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:44 smithi083 ceph-mon[24810]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:45.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:44 smithi083 ceph-mon[29774]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:45.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:44 smithi189 ceph-mon[25797]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:46.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:46 smithi083 ceph-mon[24810]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:43:46.675 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:46 smithi083 ceph-mon[29774]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:43:46.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:46 smithi189 ceph-mon[25797]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:43:49.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:48 smithi083 ceph-mon[24810]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:49.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:48 smithi083 ceph-mon[29774]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:49.192 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:48 smithi189 ceph-mon[25797]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:50.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:43:50.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:43:50.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:43:50.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:43:50.207 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:43:50.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:43:51.058 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:50 smithi189 ceph-mon[25797]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:51.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:50 smithi083 ceph-mon[24810]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:51.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:50 smithi083 ceph-mon[29774]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:52.175 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:43:51 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:43:51] "GET /metrics HTTP/1.1" 200 208045 "" "Prometheus/2.33.4" 2024-04-07T14:43:53.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:52 smithi083 ceph-mon[24810]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:43:53.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:52 smithi083 ceph-mon[29774]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:43:53.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:52 smithi189 ceph-mon[25797]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:43:53.875 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:43:53 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:43:53.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:43:53.875 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:43:53 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:43:53.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:43:53.875 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:43:53 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:43:53.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:43:54.136 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:53 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:43:54.136 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:53 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:43:54.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:53 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:43:55.127 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:54 smithi083 ceph-mon[29774]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:55.127 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:54 smithi083 ceph-mon[24810]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:55.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:54 smithi189 ceph-mon[25797]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:56.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:56 smithi083 ceph-mon[24810]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:43:56.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:56 smithi083 ceph-mon[29774]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:43:56.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:56 smithi189 ceph-mon[25797]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:43:57.139 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:43:56 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:43:56] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:43:58.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:57 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:58.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:57 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:58.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:57 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:58.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:57 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:58.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:57 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:58.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:57 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:43:59.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:43:58 smithi083 ceph-mon[24810]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:59.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:43:58 smithi083 ceph-mon[29774]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:43:59.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:43:58 smithi189 ceph-mon[25797]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:01.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:00 smithi083 ceph-mon[24810]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:01.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:00 smithi083 ceph-mon[29774]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:01.207 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:00 smithi189 ceph-mon[25797]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:02.175 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:44:01 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:44:01] "GET /metrics HTTP/1.1" 200 208045 "" "Prometheus/2.33.4" 2024-04-07T14:44:03.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:02 smithi083 ceph-mon[24810]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:44:03.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:02 smithi083 ceph-mon[29774]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:44:03.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:02 smithi189 ceph-mon[25797]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:44:03.870 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:44:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:44:03] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:44:03.871 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:44:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:44:03.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:44:03.871 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:44:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:44:03.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:44:03.871 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:44:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:44:03.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:44:04.174 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:03 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:44:04.175 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:03 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:44:04.175 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:03 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:44:04.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:03 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:44:04.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:03 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:44:04.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:03 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:44:04.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:03 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:44:04.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:03 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:44:04.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:03 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:44:05.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:04 smithi083 ceph-mon[24810]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:05.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:04 smithi083 ceph-mon[29774]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:05.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:04 smithi189 ceph-mon[25797]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:06.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:06 smithi083 ceph-mon[24810]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:44:06.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:06 smithi083 ceph-mon[29774]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:44:06.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:06 smithi189 ceph-mon[25797]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:44:07.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:07 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:44:07.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:07 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:44:07.676 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:07 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:44:07.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:07 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:44:07.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:07 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:44:07.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:07 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:44:07.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:07 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:44:07.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:07 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:44:07.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:07 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:44:08.452 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:08 smithi189 ceph-mon[25797]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:08.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:08 smithi083 ceph-mon[24810]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:08.675 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:08 smithi083 ceph-mon[29774]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:11.055 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:10 smithi189 ceph-mon[25797]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:11.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:10 smithi083 ceph-mon[24810]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:11.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:10 smithi083 ceph-mon[29774]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:12.175 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:44:11 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:44:11] "GET /metrics HTTP/1.1" 200 208029 "" "Prometheus/2.33.4" 2024-04-07T14:44:13.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:12 smithi083 ceph-mon[24810]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:44:13.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:12 smithi083 ceph-mon[29774]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:44:13.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:12 smithi189 ceph-mon[25797]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:44:13.876 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:44:13 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:44:13.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:44:13.876 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:44:13 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:44:13.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:44:13.876 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:44:13 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:44:13.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:44:14.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:13 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:44:14.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:13 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:44:14.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:13 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:44:15.095 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:14 smithi083 ceph-mon[24810]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:15.096 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:14 smithi083 ceph-mon[29774]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:15.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:14 smithi189 ceph-mon[25797]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:16.615 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:16 smithi083 ceph-mon[29774]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:44:16.615 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:16 smithi083 ceph-mon[24810]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:44:16.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:16 smithi189 ceph-mon[25797]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:44:16.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:44:16 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:44:16] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:44:18.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:17 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:44:18.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:17 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:44:18.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:17 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:44:18.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:17 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:44:18.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:17 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:44:18.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:17 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:44:19.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:18 smithi083 ceph-mon[24810]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:19.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:18 smithi083 ceph-mon[29774]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:19.207 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:18 smithi189 ceph-mon[25797]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:21.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:20 smithi083 ceph-mon[24810]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:21.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:20 smithi083 ceph-mon[29774]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:21.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:20 smithi189 ceph-mon[25797]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:21.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:44:21 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:44:21] "GET /metrics HTTP/1.1" 200 208035 "" "Prometheus/2.33.4" 2024-04-07T14:44:21.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:44:21 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:44:21.625Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:44:21.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:44:21 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:44:21.627Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:44:23.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:22 smithi083 ceph-mon[29774]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:44:23.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:22 smithi083 ceph-mon[24810]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:44:23.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:22 smithi189 ceph-mon[25797]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:44:23.824 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:44:23 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:44:23.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:44:23.824 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:44:23 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:44:23.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:44:23.824 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:44:23 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:44:23.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:44:24.175 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:44:23 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:44:23] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:44:24.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:23 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:44:24.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:23 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:44:24.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:23 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:44:24.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:23 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:44:24.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:23 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:44:24.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:23 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:44:24.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:23 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:44:24.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:23 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:44:24.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:23 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:44:25.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:24 smithi083 ceph-mon[29774]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:25.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:24 smithi083 ceph-mon[24810]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:25.207 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:24 smithi189 ceph-mon[25797]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:26.675 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:26 smithi083 ceph-mon[29774]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:44:26.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:26 smithi083 ceph-mon[24810]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:44:26.707 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:26 smithi189 ceph-mon[25797]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:44:29.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:28 smithi083 ceph-mon[24810]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:29.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:28 smithi083 ceph-mon[29774]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:29.207 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:28 smithi189 ceph-mon[25797]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:30.958 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:30 smithi189 ceph-mon[25797]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:31.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:30 smithi083 ceph-mon[24810]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:31.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:30 smithi083 ceph-mon[29774]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:31.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:44:31 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:44:31] "GET /metrics HTTP/1.1" 200 208035 "" "Prometheus/2.33.4" 2024-04-07T14:44:31.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:44:31 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:44:31.626Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:44:31.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:44:31 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:44:31.627Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:44:31.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:44:31 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:44:31.628Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:44:33.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:32 smithi083 ceph-mon[24810]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:44:33.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:32 smithi083 ceph-mon[29774]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:44:33.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:32 smithi189 ceph-mon[25797]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:44:33.883 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:44:33 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:44:33.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:44:33.883 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:44:33 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:44:33.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:44:33.883 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:44:33 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:44:33.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:44:34.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:33 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:44:34.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:33 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:44:34.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:33 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:44:35.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:34 smithi083 ceph-mon[24810]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:35.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:34 smithi083 ceph-mon[29774]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:35.207 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:34 smithi189 ceph-mon[25797]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:36.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:36 smithi083 ceph-mon[24810]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:44:36.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:36 smithi083 ceph-mon[29774]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:44:36.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:36 smithi189 ceph-mon[25797]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:44:37.164 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:44:36 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:44:36] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:44:38.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:37 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:44:38.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:37 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:44:38.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:37 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:44:38.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:37 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:44:38.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:37 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:44:38.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:37 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:44:39.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:38 smithi189 ceph-mon[25797]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:39.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:38 smithi083 ceph-mon[24810]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:39.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:38 smithi083 ceph-mon[29774]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:41.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:40 smithi189 ceph-mon[25797]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:41.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:40 smithi083 ceph-mon[24810]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:41.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:40 smithi083 ceph-mon[29774]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:41.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:44:41 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:44:41] "GET /metrics HTTP/1.1" 200 208045 "" "Prometheus/2.33.4" 2024-04-07T14:44:41.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:44:41 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:44:41.627Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:44:41.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:44:41 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:44:41.628Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:44:41.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:44:41 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:44:41.629Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:44:43.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:42 smithi189 ceph-mon[25797]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:44:43.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:42 smithi083 ceph-mon[24810]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:44:43.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:42 smithi083 ceph-mon[29774]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:44:43.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:44:43 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:44:43.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:44:43.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:44:43 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:44:43.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:44:43.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:44:43 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:44:43.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:44:44.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:43 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:44:44.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:43 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:44:44.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:43 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:44:44.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:43 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:44:44.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:43 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:44:44.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:43 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:44:44.426 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:44:43 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:44:43] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:44:44.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:43 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:44:44.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:43 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:44:44.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:43 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:44:45.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:44 smithi083 ceph-mon[24810]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:45.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:44 smithi083 ceph-mon[29774]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:45.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:44 smithi189 ceph-mon[25797]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:46.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:46 smithi083 ceph-mon[24810]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:44:46.675 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:46 smithi083 ceph-mon[29774]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:44:46.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:46 smithi189 ceph-mon[25797]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:44:49.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:48 smithi083 ceph-mon[24810]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:49.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:48 smithi083 ceph-mon[29774]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:49.207 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:48 smithi189 ceph-mon[25797]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:50.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:44:50.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:44:50.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:44:50.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:44:50.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:44:50.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:44:51.162 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:50 smithi189 ceph-mon[25797]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:51.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:50 smithi083 ceph-mon[24810]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:51.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:50 smithi083 ceph-mon[29774]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:51.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:44:51 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:44:51] "GET /metrics HTTP/1.1" 200 208039 "" "Prometheus/2.33.4" 2024-04-07T14:44:51.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:44:51 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:44:51.628Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:44:51.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:44:51 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:44:51.629Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:44:51.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:44:51 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:44:51.630Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:44:53.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:52 smithi083 ceph-mon[24810]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:44:53.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:52 smithi083 ceph-mon[29774]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:44:53.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:52 smithi189 ceph-mon[25797]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:44:53.895 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:44:53 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:44:53.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:44:53.895 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:44:53 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:44:53.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:44:53.895 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:44:53 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:44:53.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:44:54.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:53 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:44:54.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:53 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:44:54.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:53 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:44:55.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:54 smithi083 ceph-mon[24810]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:55.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:54 smithi083 ceph-mon[29774]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:55.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:54 smithi189 ceph-mon[25797]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:56.489 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:56 smithi083 ceph-mon[24810]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:44:56.490 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:56 smithi083 ceph-mon[29774]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:44:56.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:56 smithi189 ceph-mon[25797]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:44:57.138 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:44:57 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:44:57] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:44:58.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:58 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:44:58.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:58 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:44:58.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:58 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:44:58.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:58 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:44:58.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:58 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:44:58.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:58 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:44:59.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:44:59 smithi083 ceph-mon[24810]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:59.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:44:59 smithi083 ceph-mon[29774]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:44:59.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:44:59 smithi189 ceph-mon[25797]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:01.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:01 smithi083 ceph-mon[24810]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:01.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:01 smithi083 ceph-mon[29774]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:01.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:01 smithi189 ceph-mon[25797]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:01.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:45:01 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:45:01] "GET /metrics HTTP/1.1" 200 208039 "" "Prometheus/2.33.4" 2024-04-07T14:45:01.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:45:01 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:45:01.629Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:45:01.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:45:01 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:45:01.629Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:45:01.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:45:01 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:45:01.631Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:45:03.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:03 smithi083 ceph-mon[24810]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:45:03.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:03 smithi083 ceph-mon[29774]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:45:03.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:03 smithi189 ceph-mon[25797]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:45:03.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:45:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:45:03.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:45:03.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:45:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:45:03.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:45:03.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:45:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:45:03.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:45:04.425 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:45:04 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:45:04] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:45:04.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:04 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:45:04.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:04 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:45:04.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:04 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:45:04.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:04 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:45:04.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:04 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:45:04.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:04 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:45:04.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:04 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:45:04.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:04 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:45:04.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:04 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:45:05.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:05 smithi083 ceph-mon[24810]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:05.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:05 smithi083 ceph-mon[29774]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:05.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:05 smithi189 ceph-mon[25797]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:06.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:06 smithi083 ceph-mon[24810]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:45:06.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:06 smithi083 ceph-mon[29774]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:45:06.710 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:06 smithi189 ceph-mon[25797]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:45:07.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:07 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:45:07.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:07 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:45:07.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:07 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:45:07.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:07 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:45:07.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:07 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:45:07.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:07 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:45:07.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:07 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-07T14:45:07.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:07 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-07T14:45:07.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:07 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:45:08.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:08 smithi083 ceph-mon[24810]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:08.675 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:08 smithi083 ceph-mon[29774]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:08.707 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:08 smithi189 ceph-mon[25797]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:11.042 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:10 smithi189 ceph-mon[25797]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:11.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:10 smithi083 ceph-mon[24810]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:11.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:10 smithi083 ceph-mon[29774]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:11.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:45:11 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:45:11] "GET /metrics HTTP/1.1" 200 208038 "" "Prometheus/2.33.4" 2024-04-07T14:45:11.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:45:11 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:45:11.629Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:45:11.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:45:11 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:45:11.630Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:45:11.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:45:11 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:45:11.631Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:45:13.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:12 smithi083 ceph-mon[24810]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:45:13.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:12 smithi083 ceph-mon[29774]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:45:13.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:12 smithi189 ceph-mon[25797]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:45:13.894 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:45:13 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:45:13.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:45:13.894 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:45:13 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:45:13.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:45:13.894 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:45:13 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:45:13.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:45:14.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:13 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:45:14.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:13 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:45:14.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:13 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:45:15.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:14 smithi083 ceph-mon[29774]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:15.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:14 smithi083 ceph-mon[24810]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:15.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:14 smithi189 ceph-mon[25797]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:16.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:16 smithi083 ceph-mon[24810]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:45:16.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:16 smithi083 ceph-mon[29774]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:45:16.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:16 smithi189 ceph-mon[25797]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:45:17.425 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:45:17 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:45:17] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:45:18.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:18 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:45:18.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:18 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:45:18.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:18 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:45:18.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:18 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:45:18.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:18 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:45:18.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:18 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:45:19.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:19 smithi083 ceph-mon[24810]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:19.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:19 smithi083 ceph-mon[29774]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:19.455 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:19 smithi189 ceph-mon[25797]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:21.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:21 smithi083 ceph-mon[24810]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:21.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:21 smithi083 ceph-mon[29774]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:21.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:21 smithi189 ceph-mon[25797]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:21.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:45:21 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:45:21] "GET /metrics HTTP/1.1" 200 208045 "" "Prometheus/2.33.4" 2024-04-07T14:45:21.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:45:21 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:45:21.630Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:45:21.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:45:21 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:45:21.630Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:45:21.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:45:21 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:45:21.632Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:45:23.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:23 smithi083 ceph-mon[24810]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:45:23.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:23 smithi083 ceph-mon[29774]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:45:23.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:23 smithi189 ceph-mon[25797]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:45:23.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:45:23 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:45:23.517Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:45:23.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:45:23 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:45:23.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:45:23.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:45:23 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:45:23.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:45:24.425 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:45:24 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:45:24] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:45:24.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:24 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:45:24.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:24 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:45:24.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:24 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:45:24.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:24 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:45:24.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:24 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:45:24.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:24 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:45:24.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:24 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:45:24.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:24 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:45:24.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:24 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:45:25.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:25 smithi189 ceph-mon[25797]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:25.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:25 smithi083 ceph-mon[24810]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:25.675 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:25 smithi083 ceph-mon[29774]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:26.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:26 smithi083 ceph-mon[24810]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:45:26.675 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:26 smithi083 ceph-mon[29774]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:45:26.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:26 smithi189 ceph-mon[25797]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:45:29.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:28 smithi083 ceph-mon[24810]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:29.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:28 smithi083 ceph-mon[29774]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:29.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:28 smithi189 ceph-mon[25797]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:31.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:30 smithi083 ceph-mon[24810]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:31.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:30 smithi083 ceph-mon[29774]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:31.184 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:30 smithi189 ceph-mon[25797]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:31.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:45:31 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:45:31] "GET /metrics HTTP/1.1" 200 208045 "" "Prometheus/2.33.4" 2024-04-07T14:45:31.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:45:31 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:45:31.631Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:45:31.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:45:31 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:45:31.631Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:45:31.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:45:31 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:45:31.633Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:45:33.099 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:32 smithi189 ceph-mon[25797]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:45:33.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:32 smithi083 ceph-mon[24810]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:45:33.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:32 smithi083 ceph-mon[29774]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:45:33.902 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:45:33 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:45:33.517Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:45:33.902 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:45:33 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:45:33.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:45:33.902 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:45:33 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:45:33.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:45:34.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:33 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:45:34.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:33 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:45:34.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:33 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:45:35.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:34 smithi083 ceph-mon[24810]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:35.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:34 smithi083 ceph-mon[29774]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:35.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:34 smithi189 ceph-mon[25797]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:36.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:36 smithi083 ceph-mon[24810]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:45:36.675 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:36 smithi083 ceph-mon[29774]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:45:36.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:36 smithi189 ceph-mon[25797]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:45:37.425 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:45:37 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:45:37] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:45:38.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:38 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:45:38.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:38 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:45:38.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:38 smithi083 ceph-mon[24810]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:38.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:38 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:45:38.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:38 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:45:38.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:38 smithi083 ceph-mon[29774]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:38.707 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:38 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:45:38.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:38 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:45:38.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:38 smithi189 ceph-mon[25797]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:41.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:40 smithi083 ceph-mon[24810]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:41.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:40 smithi083 ceph-mon[29774]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:41.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:40 smithi189 ceph-mon[25797]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:41.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:45:41 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:45:41] "GET /metrics HTTP/1.1" 200 208042 "" "Prometheus/2.33.4" 2024-04-07T14:45:41.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:45:41 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:45:41.632Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:45:41.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:45:41 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:45:41.632Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:45:41.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:45:41 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:45:41.634Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:45:43.082 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:42 smithi083 ceph-mon[24810]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:45:43.082 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:42 smithi083 ceph-mon[29774]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:45:43.112 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -e sha1=e5c885fe95ee982ff7b45214daa61b1a6d09d57d -- bash -c 'ceph orch ps' 2024-04-07T14:45:43.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:42 smithi189 ceph-mon[25797]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:45:43.768 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:45:43 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:45:43.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:45:43.769 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:45:43 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:45:43.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:45:43.769 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:45:43 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:45:43.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:45:44.122 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:43 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:45:44.122 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:43 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:45:44.207 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:43 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:45:44.425 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:45:44 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.189 - - [07/Apr/2024:14:45:44] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:45:45.124 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:44 smithi083 ceph-mon[24810]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:45.124 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:44 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:45:45.124 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:44 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:45:45.124 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:44 smithi083 ceph-mon[29774]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:45.124 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:44 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:45:45.124 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:44 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:45:45.125 INFO:teuthology.orchestra.run.smithi083.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-07T14:45:45.125 INFO:teuthology.orchestra.run.smithi083.stdout:agent.smithi083 smithi083 running 7s ago 13m - - 2024-04-07T14:45:45.125 INFO:teuthology.orchestra.run.smithi083.stdout:agent.smithi189 smithi189 running 0s ago 13m - - 2024-04-07T14:45:45.125 INFO:teuthology.orchestra.run.smithi083.stdout:alertmanager.a smithi083 *:9093,9094 running (7m) 7s ago 7m 26.5M - ba2b418f427c 9f4fc57a8aba 2024-04-07T14:45:45.125 INFO:teuthology.orchestra.run.smithi083.stdout:grafana.a smithi189 *:3000 running (7m) 0s ago 7m 58.1M - 8.3.5 dad864ee21e9 cea72126e7bd 2024-04-07T14:45:45.125 INFO:teuthology.orchestra.run.smithi083.stdout:iscsi.foo.smithi083.qmxyme smithi083 running (6m) 7s ago 6m 56.8M - 3.5 e1d6a67b021e 81cf4f127345 2024-04-07T14:45:45.125 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.x smithi189 *:8443,9283 running (2m) 0s ago 12m 102M - 19.0.0-2578-ge5c885fe fe467d9065a1 0ef726e5100d 2024-04-07T14:45:45.125 INFO:teuthology.orchestra.run.smithi083.stdout:mgr.y smithi083 *:9283 running (14m) 7s ago 14m 471M - 17.2.0 e1d6a67b021e 966a9d7367c0 2024-04-07T14:45:45.125 INFO:teuthology.orchestra.run.smithi083.stdout:mon.a smithi083 running (14m) 7s ago 14m 54.9M 2048M 17.2.0 e1d6a67b021e 045b78f11b8e 2024-04-07T14:45:45.125 INFO:teuthology.orchestra.run.smithi083.stdout:mon.b smithi189 running (12m) 0s ago 12m 46.9M 2048M 17.2.0 e1d6a67b021e 76a325b30382 2024-04-07T14:45:45.125 INFO:teuthology.orchestra.run.smithi083.stdout:mon.c smithi083 running (12m) 7s ago 12m 43.4M 2048M 17.2.0 e1d6a67b021e 69e62d2b9b32 2024-04-07T14:45:45.125 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.a smithi083 *:9100 running (8m) 7s ago 8m 17.4M - 1dbe0e931976 a8c27029d23c 2024-04-07T14:45:45.125 INFO:teuthology.orchestra.run.smithi083.stdout:node-exporter.b smithi189 *:9100 running (8m) 0s ago 8m 17.9M - 1dbe0e931976 f7de611efab0 2024-04-07T14:45:45.126 INFO:teuthology.orchestra.run.smithi083.stdout:osd.0 smithi083 running (12m) 7s ago 12m 53.3M 4096M 17.2.0 e1d6a67b021e 8e0c47a72b20 2024-04-07T14:45:45.126 INFO:teuthology.orchestra.run.smithi083.stdout:osd.1 smithi083 running (11m) 7s ago 11m 51.2M 4096M 17.2.0 e1d6a67b021e 47f33a1b3c49 2024-04-07T14:45:45.126 INFO:teuthology.orchestra.run.smithi083.stdout:osd.2 smithi083 running (11m) 7s ago 11m 52.6M 4096M 17.2.0 e1d6a67b021e 0747abc2af4a 2024-04-07T14:45:45.126 INFO:teuthology.orchestra.run.smithi083.stdout:osd.3 smithi083 running (10m) 7s ago 10m 49.2M 4096M 17.2.0 e1d6a67b021e 57cbb5ed0f86 2024-04-07T14:45:45.126 INFO:teuthology.orchestra.run.smithi083.stdout:osd.4 smithi189 running (10m) 0s ago 10m 52.4M 4096M 17.2.0 e1d6a67b021e 27e179b6db24 2024-04-07T14:45:45.126 INFO:teuthology.orchestra.run.smithi083.stdout:osd.5 smithi189 running (10m) 0s ago 10m 48.0M 4096M 17.2.0 e1d6a67b021e 9957f4f08073 2024-04-07T14:45:45.126 INFO:teuthology.orchestra.run.smithi083.stdout:osd.6 smithi189 running (9m) 0s ago 9m 46.1M 4096M 17.2.0 e1d6a67b021e af997398de2d 2024-04-07T14:45:45.126 INFO:teuthology.orchestra.run.smithi083.stdout:osd.7 smithi189 running (9m) 0s ago 9m 49.8M 4096M 17.2.0 e1d6a67b021e 8a39104a15b0 2024-04-07T14:45:45.126 INFO:teuthology.orchestra.run.smithi083.stdout:prometheus.a smithi189 *:9095 running (7m) 0s ago 8m 69.3M - 514e6a882f6e 1aeb972809bd 2024-04-07T14:45:45.126 INFO:teuthology.orchestra.run.smithi083.stdout:rgw.foo.smithi083.oslioj smithi083 *:8000 running (6m) 7s ago 6m 69.7M - 17.2.0 e1d6a67b021e 7c352b07e177 2024-04-07T14:45:45.126 INFO:teuthology.orchestra.run.smithi083.stdout:rgw.foo.smithi189.taxdrv smithi189 *:8000 running (6m) 0s ago 6m 72.9M - 17.2.0 e1d6a67b021e ce3b15242448 2024-04-07T14:45:45.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:44 smithi189 ceph-mon[25797]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:45.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:44 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:45:45.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:44 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:45:45.879 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -e sha1=e5c885fe95ee982ff7b45214daa61b1a6d09d57d -- bash -c 'ceph versions' 2024-04-07T14:45:46.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:45 smithi083 ceph-mon[24810]: from='client.15045 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:45:46.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:45 smithi083 ceph-mon[29774]: from='client.15045 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:45:46.207 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:45 smithi189 ceph-mon[25797]: from='client.15045 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-07T14:45:47.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:46 smithi083 ceph-mon[24810]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:45:47.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:46 smithi083 ceph-mon[29774]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:45:47.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:46 smithi189 ceph-mon[25797]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:45:47.924 INFO:teuthology.orchestra.run.smithi083.stdout:{ 2024-04-07T14:45:47.924 INFO:teuthology.orchestra.run.smithi083.stdout: "mon": { 2024-04-07T14:45:47.924 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-07T14:45:47.924 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-04-07T14:45:47.924 INFO:teuthology.orchestra.run.smithi083.stdout: "mgr": { 2024-04-07T14:45:47.924 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-07T14:45:47.925 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-2578-ge5c885fe (e5c885fe95ee982ff7b45214daa61b1a6d09d57d) squid (dev)": 1 2024-04-07T14:45:47.925 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-04-07T14:45:47.925 INFO:teuthology.orchestra.run.smithi083.stdout: "osd": { 2024-04-07T14:45:47.925 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-07T14:45:47.925 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-04-07T14:45:47.925 INFO:teuthology.orchestra.run.smithi083.stdout: "mds": {}, 2024-04-07T14:45:47.925 INFO:teuthology.orchestra.run.smithi083.stdout: "rgw": { 2024-04-07T14:45:47.925 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-07T14:45:47.925 INFO:teuthology.orchestra.run.smithi083.stdout: }, 2024-04-07T14:45:47.925 INFO:teuthology.orchestra.run.smithi083.stdout: "overall": { 2024-04-07T14:45:47.925 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14, 2024-04-07T14:45:47.925 INFO:teuthology.orchestra.run.smithi083.stdout: "ceph version 19.0.0-2578-ge5c885fe (e5c885fe95ee982ff7b45214daa61b1a6d09d57d) squid (dev)": 1 2024-04-07T14:45:47.925 INFO:teuthology.orchestra.run.smithi083.stdout: } 2024-04-07T14:45:47.925 INFO:teuthology.orchestra.run.smithi083.stdout:} 2024-04-07T14:45:48.637 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -e sha1=e5c885fe95ee982ff7b45214daa61b1a6d09d57d -- bash -c 'ceph -s' 2024-04-07T14:45:49.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:48 smithi083 ceph-mon[24810]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:49.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:48 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/995839189' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-07T14:45:49.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:48 smithi083 ceph-mon[29774]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:49.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:48 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/995839189' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-07T14:45:49.207 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:48 smithi189 ceph-mon[25797]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:49.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:48 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/995839189' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-07T14:45:50.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:45:50.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:49 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:45:50.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:45:50.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:49 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:45:50.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-07T14:45:50.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:49 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-07T14:45:50.717 INFO:teuthology.orchestra.run.smithi083.stdout: cluster: 2024-04-07T14:45:50.717 INFO:teuthology.orchestra.run.smithi083.stdout: id: 7388281a-f4eb-11ee-b649-cb9ed24678a4 2024-04-07T14:45:50.717 INFO:teuthology.orchestra.run.smithi083.stdout: health: HEALTH_OK 2024-04-07T14:45:50.717 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-07T14:45:50.717 INFO:teuthology.orchestra.run.smithi083.stdout: services: 2024-04-07T14:45:50.717 INFO:teuthology.orchestra.run.smithi083.stdout: mon: 3 daemons, quorum a,c,b (age 12m) 2024-04-07T14:45:50.718 INFO:teuthology.orchestra.run.smithi083.stdout: mgr: y(active, since 9m), standbys: x 2024-04-07T14:45:50.718 INFO:teuthology.orchestra.run.smithi083.stdout: osd: 8 osds: 8 up (since 9m), 8 in (since 9m) 2024-04-07T14:45:50.718 INFO:teuthology.orchestra.run.smithi083.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-04-07T14:45:50.718 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-07T14:45:50.718 INFO:teuthology.orchestra.run.smithi083.stdout: data: 2024-04-07T14:45:50.718 INFO:teuthology.orchestra.run.smithi083.stdout: pools: 6 pools, 161 pgs 2024-04-07T14:45:50.718 INFO:teuthology.orchestra.run.smithi083.stdout: objects: 209 objects, 585 KiB 2024-04-07T14:45:50.718 INFO:teuthology.orchestra.run.smithi083.stdout: usage: 64 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:45:50.718 INFO:teuthology.orchestra.run.smithi083.stdout: pgs: 161 active+clean 2024-04-07T14:45:50.718 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-07T14:45:50.718 INFO:teuthology.orchestra.run.smithi083.stdout: io: 2024-04-07T14:45:50.718 INFO:teuthology.orchestra.run.smithi083.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-04-07T14:45:50.718 INFO:teuthology.orchestra.run.smithi083.stdout: 2024-04-07T14:45:51.124 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:50 smithi083 ceph-mon[24810]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:51.125 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:50 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/2612947945' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-07T14:45:51.125 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:50 smithi083 ceph-mon[29774]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:51.125 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:50 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/2612947945' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-07T14:45:51.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:50 smithi189 ceph-mon[25797]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:51.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:50 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/2612947945' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-07T14:45:51.423 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -e sha1=e5c885fe95ee982ff7b45214daa61b1a6d09d57d -- bash -c 'ceph health detail' 2024-04-07T14:45:51.675 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:45:51 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:45:51.632Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:45:51.676 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:45:51 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:45:51.634Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:45:51.676 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:45:51 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:45:51.635Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:45:51.983 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:45:51 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:45:51] "GET /metrics HTTP/1.1" 200 208041 "" "Prometheus/2.33.4" 2024-04-07T14:45:53.076 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:52 smithi083 ceph-mon[24810]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:45:53.076 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:52 smithi083 ceph-mon[29774]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:45:53.202 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:52 smithi189 ceph-mon[25797]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:45:53.516 INFO:teuthology.orchestra.run.smithi083.stdout:HEALTH_OK 2024-04-07T14:45:53.894 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:45:53 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:45:53.519Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:45:53.894 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:45:53 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:45:53.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:45:53.894 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:45:53 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:45:53.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:45:54.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:53 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:45:54.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:53 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/1649968068' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-07T14:45:54.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:53 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:45:54.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:53 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/1649968068' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-07T14:45:54.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:53 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:45:54.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:53 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/1649968068' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-07T14:45:54.360 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -e sha1=e5c885fe95ee982ff7b45214daa61b1a6d09d57d -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 2'"'"'' 2024-04-07T14:45:55.033 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:54 smithi189 ceph-mon[25797]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:55.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:54 smithi083 ceph-mon[24810]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:55.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:54 smithi083 ceph-mon[29774]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:56.550 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:56 smithi083 ceph-mon[29774]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:45:56.550 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:56 smithi083 ceph-mon[24810]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:45:56.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:56 smithi189 ceph-mon[25797]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:45:57.481 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:45:57 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 172.21.15.83 - - [07/Apr/2024:14:45:57] "POST /data HTTP/1.1" 200 46 "" "Python-urllib/3.9" 2024-04-07T14:45:57.493 INFO:teuthology.orchestra.run.smithi083.stdout:true 2024-04-07T14:45:58.614 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -e sha1=e5c885fe95ee982ff7b45214daa61b1a6d09d57d -- bash -c 'ceph mgr fail' 2024-04-07T14:45:58.644 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:58 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:45:58.644 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:58 smithi083 ceph-mon[24810]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:45:58.644 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:58 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/929265601' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-07T14:45:58.644 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:45:58 smithi083 ceph-mon[24810]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:58.644 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:58 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:45:58.644 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:58 smithi083 ceph-mon[29774]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:45:58.644 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:58 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/929265601' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-07T14:45:58.644 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:45:58 smithi083 ceph-mon[29774]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:45:58.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:58 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:45:58.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:58 smithi189 ceph-mon[25797]: from='mgr.24358 172.21.15.83:0/3598248665' entity='mgr.y' 2024-04-07T14:45:58.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:58 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/929265601' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-07T14:45:58.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:45:58 smithi189 ceph-mon[25797]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:46:01.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:00 smithi083 ceph-mon[24810]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:46:01.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:00 smithi083 ceph-mon[29774]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:46:01.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:00 smithi189 ceph-mon[25797]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:46:01.899 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:46:01 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:46:01] "GET /metrics HTTP/1.1" 200 208041 "" "Prometheus/2.33.4" 2024-04-07T14:46:01.899 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:46:01 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:46:01.634Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:46:01.899 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:46:01 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:46:01.635Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:46:01.900 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:46:01 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:46:01.636Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:46:02.158 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:01 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/2821323474' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-07T14:46:02.158 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:01 smithi083 ceph-mon[24810]: osdmap e97: 8 total, 8 up, 8 in 2024-04-07T14:46:02.158 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:01 smithi083 ceph-mon[24810]: Standby manager daemon y started 2024-04-07T14:46:02.159 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:01 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/2821323474' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-07T14:46:02.159 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:01 smithi083 ceph-mon[29774]: osdmap e97: 8 total, 8 up, 8 in 2024-04-07T14:46:02.159 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:01 smithi083 ceph-mon[29774]: Standby manager daemon y started 2024-04-07T14:46:02.159 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:46:01 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:46:01.911+0000 7f01059f5700 -1 mgr handle_mgr_map I was active but no longer amignoring --setuser ceph since I am not root 2024-04-07T14:46:02.159 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:46:01 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ignoring --setgroup ceph since I am not root 2024-04-07T14:46:02.159 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:46:01 smithi083 ceph-mgr[25049]: -- 172.21.15.83:0/3547585834 <== mon.0 v2:172.21.15.83:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x563bb1950340 con 0x563bb1928c00 2024-04-07T14:46:02.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:01 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/2821323474' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-07T14:46:02.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:01 smithi189 ceph-mon[25797]: osdmap e97: 8 total, 8 up, 8 in 2024-04-07T14:46:02.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:01 smithi189 ceph-mon[25797]: Standby manager daemon y started 2024-04-07T14:46:02.425 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:46:02.160Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=2 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": dial tcp 172.21.15.83:8443: connect: connection refused" 2024-04-07T14:46:02.567 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -e sha1=e5c885fe95ee982ff7b45214daa61b1a6d09d57d -- bash -c 'sleep 180' 2024-04-07T14:46:02.909 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:46:02.427+0000 7fb851581000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-07T14:46:02.909 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:46:02.517+0000 7fb851581000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-07T14:46:03.160 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:46:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:46:03.131+0000 7fb851581000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-07T14:46:03.160 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/2821323474' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-07T14:46:03.160 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-mon[24810]: mgrmap e22: x(active, starting, since 0.896506s), standbys: y 2024-04-07T14:46:03.160 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-mon[24810]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-07T14:46:03.160 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-mon[24810]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-07T14:46:03.160 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-mon[24810]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-07T14:46:03.160 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-mon[24810]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-07T14:46:03.161 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-mon[24810]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-07T14:46:03.161 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-mon[24810]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-07T14:46:03.161 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-mon[24810]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-07T14:46:03.161 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-mon[24810]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-07T14:46:03.161 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-mon[24810]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-07T14:46:03.161 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-mon[24810]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-07T14:46:03.161 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-mon[24810]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-07T14:46:03.161 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-mon[24810]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-07T14:46:03.161 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-mon[24810]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-07T14:46:03.161 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-mon[24810]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-07T14:46:03.161 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-mon[24810]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-07T14:46:03.161 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-mon[24810]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-07T14:46:03.161 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-mon[24810]: Manager daemon x is now available 2024-04-07T14:46:03.162 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/2821323474' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-07T14:46:03.162 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-mon[29774]: mgrmap e22: x(active, starting, since 0.896506s), standbys: y 2024-04-07T14:46:03.162 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-mon[29774]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-07T14:46:03.162 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-mon[29774]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-07T14:46:03.162 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-mon[29774]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-07T14:46:03.162 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-mon[29774]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-07T14:46:03.162 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-mon[29774]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-07T14:46:03.162 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-mon[29774]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-07T14:46:03.162 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-mon[29774]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-07T14:46:03.162 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-mon[29774]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-07T14:46:03.162 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-mon[29774]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-07T14:46:03.162 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-mon[29774]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-07T14:46:03.163 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-mon[29774]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-07T14:46:03.163 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-mon[29774]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-07T14:46:03.163 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-mon[29774]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-07T14:46:03.163 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-mon[29774]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-07T14:46:03.163 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-mon[29774]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-07T14:46:03.163 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-mon[29774]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-07T14:46:03.163 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:02 smithi083 ceph-mon[29774]: Manager daemon x is now available 2024-04-07T14:46:03.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:02 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/2821323474' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-07T14:46:03.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:02 smithi189 ceph-mon[25797]: mgrmap e22: x(active, starting, since 0.896506s), standbys: y 2024-04-07T14:46:03.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:02 smithi189 ceph-mon[25797]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-07T14:46:03.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:02 smithi189 ceph-mon[25797]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-07T14:46:03.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:02 smithi189 ceph-mon[25797]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-07T14:46:03.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:02 smithi189 ceph-mon[25797]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-07T14:46:03.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:02 smithi189 ceph-mon[25797]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-07T14:46:03.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:02 smithi189 ceph-mon[25797]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-07T14:46:03.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:02 smithi189 ceph-mon[25797]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-07T14:46:03.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:02 smithi189 ceph-mon[25797]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-07T14:46:03.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:02 smithi189 ceph-mon[25797]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-07T14:46:03.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:02 smithi189 ceph-mon[25797]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-07T14:46:03.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:02 smithi189 ceph-mon[25797]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-07T14:46:03.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:02 smithi189 ceph-mon[25797]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-07T14:46:03.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:02 smithi189 ceph-mon[25797]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-07T14:46:03.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:02 smithi189 ceph-mon[25797]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-07T14:46:03.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:02 smithi189 ceph-mon[25797]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-07T14:46:03.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:02 smithi189 ceph-mon[25797]: from='mgr.24722 172.21.15.189:0/905125132' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-07T14:46:03.209 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:02 smithi189 ceph-mon[25797]: Manager daemon x is now available 2024-04-07T14:46:03.425 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:46:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:46:03.225+0000 7fb851581000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-07T14:46:03.425 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:46:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:46:03.311+0000 7fb851581000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-07T14:46:03.737 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:46:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:46:03.461+0000 7fb851581000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-07T14:46:03.738 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:46:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:46:03.547+0000 7fb851581000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-07T14:46:03.738 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:46:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:46:03.656+0000 7fb851581000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-07T14:46:03.738 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:46:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:46:03.739+0000 7fb851581000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-07T14:46:03.738 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:46:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:46:03.520Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:46:03.738 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:46:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:46:03.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": dial tcp 172.21.15.83:8443: connect: connection refused" 2024-04-07T14:46:03.738 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:46:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:46:03.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:46:03.998 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:46:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:46:03.905+0000 7fb851581000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-07T14:46:03.999 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:03 smithi083 ceph-mon[24810]: mgrmap e23: x(active, since 1.89994s), standbys: y 2024-04-07T14:46:03.999 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:03 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:46:03.999 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:03 smithi083 ceph-mon[24810]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:46:03.999 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:03 smithi083 ceph-mon[29774]: mgrmap e23: x(active, since 1.89994s), standbys: y 2024-04-07T14:46:03.999 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:03 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:46:03.999 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:03 smithi083 ceph-mon[29774]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:46:04.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:03 smithi189 ceph-mon[25797]: mgrmap e23: x(active, since 1.89994s), standbys: y 2024-04-07T14:46:04.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:03 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:46:04.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:03 smithi189 ceph-mon[25797]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:46:04.254 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:46:04 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:46:04.000+0000 7fb851581000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-07T14:46:05.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:04 smithi083 ceph-mon[24810]: Health check failed: 16 mgr modules have failed dependencies (MGR_MODULE_DEPENDENCY) 2024-04-07T14:46:05.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:04 smithi083 ceph-mon[24810]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:46:05.175 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:46:05 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:46:05.025+0000 7fb851581000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-07T14:46:05.176 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:46:05 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:46:05.151+0000 7fb851581000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-07T14:46:05.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:04 smithi083 ceph-mon[29774]: Health check failed: 16 mgr modules have failed dependencies (MGR_MODULE_DEPENDENCY) 2024-04-07T14:46:05.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:04 smithi083 ceph-mon[29774]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:46:05.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:04 smithi189 ceph-mon[25797]: Health check failed: 16 mgr modules have failed dependencies (MGR_MODULE_DEPENDENCY) 2024-04-07T14:46:05.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:04 smithi189 ceph-mon[25797]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:46:05.578 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:46:05 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:46:05.346+0000 7fb851581000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-07T14:46:05.578 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:46:05 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:46:05.439+0000 7fb851581000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-07T14:46:05.578 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:46:05 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:46:05.580+0000 7fb851581000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-07T14:46:06.175 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:46:05 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:46:05.914+0000 7fb851581000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-07T14:46:06.175 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:46:06 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:46:06.143+0000 7fb851581000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-07T14:46:06.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:06 smithi083 ceph-mon[24810]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:46:06.675 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:46:06 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:46:06.230+0000 7fb851581000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-07T14:46:06.675 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:46:06 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:46:06.406+0000 7fb851581000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-07T14:46:06.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:06 smithi083 ceph-mon[29774]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:46:06.707 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:06 smithi189 ceph-mon[25797]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:46:07.611 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:46:07 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:46:07.320+0000 7fb851581000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-07T14:46:07.874 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:46:07 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:46:07.613+0000 7fb851581000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-07T14:46:07.874 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:46:07 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:46:07.709+0000 7fb851581000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-07T14:46:07.874 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:46:07 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:46:07.860+0000 7fb851581000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-07T14:46:07.874 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:46:07 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: [07/Apr/2024:14:46:07] ENGINE Bus STARTING 2024-04-07T14:46:08.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:07 smithi083 ceph-mon[24810]: Standby manager daemon y restarted 2024-04-07T14:46:08.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:07 smithi083 ceph-mon[24810]: Standby manager daemon y started 2024-04-07T14:46:08.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:07 smithi083 ceph-mon[24810]: from='mgr.? 172.21.15.83:0/1723242073' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-07T14:46:08.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:07 smithi083 ceph-mon[24810]: from='mgr.? 172.21.15.83:0/1723242073' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-07T14:46:08.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:07 smithi083 ceph-mon[24810]: from='mgr.? 172.21.15.83:0/1723242073' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-07T14:46:08.176 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:07 smithi083 ceph-mon[24810]: from='mgr.? 172.21.15.83:0/1723242073' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-07T14:46:08.176 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:46:07 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: CherryPy Checker: 2024-04-07T14:46:08.176 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:46:07 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: The Application mounted at '' has an empty config. 2024-04-07T14:46:08.176 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:46:07 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: 2024-04-07T14:46:08.176 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:46:07 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: [07/Apr/2024:14:46:07] ENGINE Serving on http://:::9283 2024-04-07T14:46:08.176 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:46:07 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: [07/Apr/2024:14:46:07] ENGINE Bus STARTED 2024-04-07T14:46:08.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:07 smithi083 ceph-mon[29774]: Standby manager daemon y restarted 2024-04-07T14:46:08.176 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:07 smithi083 ceph-mon[29774]: Standby manager daemon y started 2024-04-07T14:46:08.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:07 smithi083 ceph-mon[29774]: from='mgr.? 172.21.15.83:0/1723242073' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-07T14:46:08.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:07 smithi083 ceph-mon[29774]: from='mgr.? 172.21.15.83:0/1723242073' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-07T14:46:08.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:07 smithi083 ceph-mon[29774]: from='mgr.? 172.21.15.83:0/1723242073' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-07T14:46:08.177 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:07 smithi083 ceph-mon[29774]: from='mgr.? 172.21.15.83:0/1723242073' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-07T14:46:08.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:07 smithi189 ceph-mon[25797]: Standby manager daemon y restarted 2024-04-07T14:46:08.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:07 smithi189 ceph-mon[25797]: Standby manager daemon y started 2024-04-07T14:46:08.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:07 smithi189 ceph-mon[25797]: from='mgr.? 172.21.15.83:0/1723242073' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-07T14:46:08.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:07 smithi189 ceph-mon[25797]: from='mgr.? 172.21.15.83:0/1723242073' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-07T14:46:08.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:07 smithi189 ceph-mon[25797]: from='mgr.? 172.21.15.83:0/1723242073' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-07T14:46:08.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:07 smithi189 ceph-mon[25797]: from='mgr.? 172.21.15.83:0/1723242073' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-07T14:46:09.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:08 smithi083 ceph-mon[29774]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:46:09.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:08 smithi083 ceph-mon[29774]: mgrmap e24: x(active, since 6s), standbys: y 2024-04-07T14:46:09.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:08 smithi083 ceph-mon[24810]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:46:09.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:08 smithi083 ceph-mon[24810]: mgrmap e24: x(active, since 6s), standbys: y 2024-04-07T14:46:09.176 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:46:08 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:46:08.719Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=6 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:46:09.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:08 smithi189 ceph-mon[25797]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail 2024-04-07T14:46:09.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:08 smithi189 ceph-mon[25797]: mgrmap e24: x(active, since 6s), standbys: y 2024-04-07T14:46:10.675 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:46:10 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:46:10.371Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=6 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:46:11.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:10 smithi189 ceph-mon[25797]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 731 B/s rd, 0 op/s 2024-04-07T14:46:11.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:10 smithi083 ceph-mon[24810]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 731 B/s rd, 0 op/s 2024-04-07T14:46:11.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:10 smithi083 ceph-mon[29774]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 731 B/s rd, 0 op/s 2024-04-07T14:46:11.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:46:11 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:46:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:46:11.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:46:11 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:46:11.635Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:46:11.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:46:11 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:46:11.636Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:46:11.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:46:11 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:46:11.637Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:46:13.207 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:12 smithi189 ceph-mon[25797]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 569 B/s rd, 0 op/s 2024-04-07T14:46:13.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:12 smithi083 ceph-mon[24810]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 569 B/s rd, 0 op/s 2024-04-07T14:46:13.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:12 smithi083 ceph-mon[29774]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 569 B/s rd, 0 op/s 2024-04-07T14:46:13.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:46:13 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:46:13.521Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:46:13.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:46:13 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:46:13.522Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:46:13.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:46:13 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:46:13.524Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:46:14.207 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:13 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:46:14.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:13 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:46:14.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:13 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:46:15.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:14 smithi189 ceph-mon[25797]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 931 B/s rd, 0 op/s 2024-04-07T14:46:15.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:14 smithi083 ceph-mon[24810]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 931 B/s rd, 0 op/s 2024-04-07T14:46:15.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:14 smithi083 ceph-mon[29774]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 931 B/s rd, 0 op/s 2024-04-07T14:46:16.589 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:16 smithi083 ceph-mon[24810]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:46:16.589 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:16 smithi083 ceph-mon[29774]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:46:16.707 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:16 smithi189 ceph-mon[25797]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:46:19.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:18 smithi083 ceph-mon[24810]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:46:19.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:18 smithi083 ceph-mon[29774]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:46:19.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:18 smithi189 ceph-mon[25797]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:46:21.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:20 smithi083 ceph-mon[24810]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:46:21.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:20 smithi083 ceph-mon[29774]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:46:21.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:20 smithi189 ceph-mon[25797]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:46:21.926 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:46:21 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:46:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:46:21.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:46:21 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:46:21.636Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:46:21.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:46:21 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:46:21.637Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:46:21.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:46:21 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:46:21.639Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:46:23.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:22 smithi083 ceph-mon[24810]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:46:23.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:22 smithi083 ceph-mon[29774]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:46:23.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:22 smithi189 ceph-mon[25797]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:46:23.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:46:23 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:46:23.521Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:46:23.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:46:23 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:46:23.522Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:46:23.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:46:23 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:46:23.523Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:46:24.419 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:23 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:46:24.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:23 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:46:24.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:23 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:46:25.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:24 smithi083 ceph-mon[24810]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:46:25.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:24 smithi083 ceph-mon[29774]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:46:25.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:24 smithi189 ceph-mon[25797]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:46:26.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:26 smithi083 ceph-mon[24810]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:46:26.675 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:26 smithi083 ceph-mon[29774]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:46:26.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:26 smithi189 ceph-mon[25797]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:46:29.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:28 smithi083 ceph-mon[24810]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:46:29.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:28 smithi083 ceph-mon[29774]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:46:29.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:28 smithi189 ceph-mon[25797]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:46:31.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:30 smithi083 ceph-mon[24810]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:46:31.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:30 smithi083 ceph-mon[29774]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:46:31.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:30 smithi189 ceph-mon[25797]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:46:31.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:46:31 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:46:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:46:31.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:46:31 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:46:31.638Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:46:31.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:46:31 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:46:31.638Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:46:31.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:46:31 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:46:31.640Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:46:33.255 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:32 smithi189 ceph-mon[25797]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:46:33.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:32 smithi083 ceph-mon[24810]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:46:33.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:32 smithi083 ceph-mon[29774]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:46:33.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:46:33 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:46:33.522Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:46:33.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:46:33 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:46:33.523Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:46:33.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:46:33 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:46:33.527Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:46:34.318 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:33 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:46:34.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:33 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:46:34.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:33 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:46:35.248 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:34 smithi189 ceph-mon[25797]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:46:35.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:34 smithi083 ceph-mon[24810]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:46:35.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:34 smithi083 ceph-mon[29774]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:46:36.553 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:36 smithi083 ceph-mon[24810]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:46:36.554 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:36 smithi083 ceph-mon[29774]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:46:36.707 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:36 smithi189 ceph-mon[25797]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:46:39.385 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:38 smithi083 ceph-mon[24810]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:46:39.385 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:38 smithi083 ceph-mon[29774]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:46:39.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:38 smithi189 ceph-mon[25797]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:46:41.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:40 smithi083 ceph-mon[24810]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:46:41.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:40 smithi083 ceph-mon[29774]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:46:41.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:40 smithi189 ceph-mon[25797]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:46:41.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:46:41 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:46:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:46:41.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:46:41 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:46:41.638Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:46:41.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:46:41 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:46:41.639Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:46:41.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:46:41 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:46:41.640Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:46:43.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:42 smithi083 ceph-mon[24810]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:46:43.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:42 smithi083 ceph-mon[29774]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:46:43.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:42 smithi189 ceph-mon[25797]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:46:43.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:46:43 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:46:43.523Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:46:43.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:46:43 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:46:43.524Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:46:43.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:46:43 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:46:43.526Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:46:44.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:44 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:46:44.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:44 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:46:44.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:44 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:46:45.301 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:45 smithi189 ceph-mon[25797]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:46:45.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:45 smithi083 ceph-mon[24810]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:46:45.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:45 smithi083 ceph-mon[29774]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:46:46.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:46 smithi083 ceph-mon[24810]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:46:46.675 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:46 smithi083 ceph-mon[29774]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:46:46.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:46 smithi189 ceph-mon[25797]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:46:49.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:48 smithi083 ceph-mon[24810]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:46:49.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:48 smithi083 ceph-mon[29774]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:46:49.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:48 smithi189 ceph-mon[25797]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:46:51.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:50 smithi083 ceph-mon[24810]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-07T14:46:51.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:50 smithi083 ceph-mon[29774]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-07T14:46:51.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:50 smithi189 ceph-mon[25797]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-07T14:46:51.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:46:51 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:46:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:46:51.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:46:51 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:46:51.639Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:46:51.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:46:51 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:46:51.640Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:46:51.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:46:51 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:46:51.641Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:46:53.103 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:52 smithi189 ceph-mon[25797]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-07T14:46:53.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:52 smithi083 ceph-mon[24810]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-07T14:46:53.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:52 smithi083 ceph-mon[29774]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-07T14:46:53.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:46:53 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:46:53.524Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:46:53.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:46:53 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:46:53.525Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:46:53.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:46:53 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:46:53.527Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:46:54.187 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:54 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:46:54.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:54 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:46:54.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:54 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:46:55.260 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:55 smithi189 ceph-mon[25797]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:46:55.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:55 smithi083 ceph-mon[24810]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:46:55.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:55 smithi083 ceph-mon[29774]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:46:56.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:56 smithi189 ceph-mon[25797]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-07T14:46:56.645 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:56 smithi083 ceph-mon[24810]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-07T14:46:56.645 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:56 smithi083 ceph-mon[29774]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-07T14:46:59.283 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:46:58 smithi083 ceph-mon[24810]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:46:59.283 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:46:58 smithi083 ceph-mon[29774]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:46:59.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:46:58 smithi189 ceph-mon[25797]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:47:01.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:47:00 smithi083 ceph-mon[24810]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:01.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:47:00 smithi083 ceph-mon[29774]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:01.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:47:00 smithi189 ceph-mon[25797]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:01.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:47:01 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:47:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:47:01.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:47:01 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:47:01.640Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:47:01.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:47:01 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:47:01.641Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:47:01.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:47:01 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:47:01.643Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:47:03.260 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:47:02 smithi189 ceph-mon[25797]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:03.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:47:02 smithi083 ceph-mon[24810]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:03.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:47:02 smithi083 ceph-mon[29774]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:03.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:47:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:47:03.525Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:47:03.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:47:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:47:03.526Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:47:03.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:47:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:47:03.527Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:47:04.271 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:47:04 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:47:04.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:47:04 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:47:04.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:47:04 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:47:05.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:47:05 smithi083 ceph-mon[24810]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:47:05.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:47:05 smithi083 ceph-mon[29774]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:47:05.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:47:05 smithi189 ceph-mon[25797]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:47:06.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:47:06 smithi083 ceph-mon[24810]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:06.675 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:47:06 smithi083 ceph-mon[29774]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:06.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:47:06 smithi189 ceph-mon[25797]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:09.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:47:08 smithi083 ceph-mon[24810]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:47:09.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:47:08 smithi083 ceph-mon[29774]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:47:09.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:47:08 smithi189 ceph-mon[25797]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:47:11.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:47:10 smithi083 ceph-mon[24810]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:11.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:47:11 smithi083 ceph-mon[29774]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:11.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:47:10 smithi189 ceph-mon[25797]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:11.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:47:11 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:47:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:47:11.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:47:11 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:47:11.641Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:47:11.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:47:11 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:47:11.642Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:47:11.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:47:11 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:47:11.643Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:47:13.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:47:13 smithi083 ceph-mon[24810]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:13.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:47:13 smithi083 ceph-mon[29774]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:13.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:47:13 smithi189 ceph-mon[25797]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:13.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:47:13 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:47:13.526Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:47:13.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:47:13 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:47:13.527Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:47:13.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:47:13 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:47:13.528Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:47:14.310 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:47:14 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:47:14.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:47:14 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:47:14.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:47:14 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:47:15.307 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:47:15 smithi189 ceph-mon[25797]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:47:15.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:47:15 smithi083 ceph-mon[24810]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:47:15.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:47:15 smithi083 ceph-mon[29774]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:47:16.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:47:16 smithi189 ceph-mon[25797]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:16.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:47:16 smithi083 ceph-mon[24810]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:16.675 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:47:16 smithi083 ceph-mon[29774]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:19.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:47:18 smithi083 ceph-mon[24810]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:47:19.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:47:18 smithi083 ceph-mon[29774]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:47:19.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:47:18 smithi189 ceph-mon[25797]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:47:21.207 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:47:21 smithi189 ceph-mon[25797]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:21.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:47:21 smithi083 ceph-mon[24810]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:21.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:47:21 smithi083 ceph-mon[29774]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:21.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:47:21 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:47:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:47:21.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:47:21 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:47:21.642Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:47:21.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:47:21 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:47:21.643Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:47:21.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:47:21 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:47:21.644Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:47:23.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:47:23 smithi083 ceph-mon[24810]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:23.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:47:23 smithi083 ceph-mon[29774]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:23.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:47:23 smithi189 ceph-mon[25797]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:23.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:47:23 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:47:23.527Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:47:23.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:47:23 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:47:23.528Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:47:23.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:47:23 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:47:23.529Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:47:24.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:47:24 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:47:24.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:47:24 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:47:24.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:47:24 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:47:25.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:47:25 smithi083 ceph-mon[24810]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:47:25.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:47:25 smithi083 ceph-mon[29774]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:47:25.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:47:25 smithi189 ceph-mon[25797]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:47:26.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:47:26 smithi083 ceph-mon[24810]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:26.675 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:47:26 smithi083 ceph-mon[29774]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:26.707 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:47:26 smithi189 ceph-mon[25797]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:29.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:47:29 smithi083 ceph-mon[24810]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:47:29.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:47:29 smithi083 ceph-mon[29774]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:47:29.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:47:29 smithi189 ceph-mon[25797]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:47:31.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:47:31 smithi083 ceph-mon[24810]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:31.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:47:31 smithi083 ceph-mon[29774]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:31.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:47:31 smithi189 ceph-mon[25797]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:31.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:47:31 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:47:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:47:31.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:47:31 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:47:31.643Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:47:31.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:47:31 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:47:31.644Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:47:31.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:47:31 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:47:31.645Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:47:33.260 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:47:33 smithi189 ceph-mon[25797]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:33.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:47:33 smithi083 ceph-mon[24810]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:33.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:47:33 smithi083 ceph-mon[29774]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:33.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:47:33 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:47:33.528Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:47:33.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:47:33 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:47:33.528Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:47:33.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:47:33 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:47:33.530Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:47:34.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:47:34 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:47:34.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:47:34 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:47:34.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:47:34 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:47:35.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:47:35 smithi083 ceph-mon[24810]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:47:35.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:47:35 smithi083 ceph-mon[29774]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:47:35.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:47:35 smithi189 ceph-mon[25797]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:47:36.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:47:36 smithi083 ceph-mon[24810]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:36.675 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:47:36 smithi083 ceph-mon[29774]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:36.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:47:36 smithi189 ceph-mon[25797]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:39.297 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:47:39 smithi083 ceph-mon[24810]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:47:39.297 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:47:39 smithi083 ceph-mon[29774]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:47:39.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:47:39 smithi189 ceph-mon[25797]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:47:41.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:47:41 smithi083 ceph-mon[24810]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:41.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:47:41 smithi083 ceph-mon[29774]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:41.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:47:41 smithi189 ceph-mon[25797]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:41.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:47:41 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:47:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:47:41.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:47:41 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:47:41.644Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:47:41.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:47:41 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:47:41.644Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:47:41.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:47:41 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:47:41.646Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:47:43.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:47:43 smithi083 ceph-mon[24810]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:43.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:47:43 smithi083 ceph-mon[29774]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:43.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:47:43 smithi189 ceph-mon[25797]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:43.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:47:43 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:47:43.529Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:47:43.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:47:43 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:47:43.530Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:47:43.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:47:43 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:47:43.531Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:47:44.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:47:44 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:47:44.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:47:44 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:47:44.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:47:44 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:47:45.207 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:47:45 smithi189 ceph-mon[25797]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:47:45.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:47:45 smithi083 ceph-mon[24810]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:47:45.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:47:45 smithi083 ceph-mon[29774]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:47:46.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:47:46 smithi083 ceph-mon[24810]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:46.675 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:47:46 smithi083 ceph-mon[29774]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:46.707 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:47:46 smithi189 ceph-mon[25797]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:49.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:47:49 smithi083 ceph-mon[24810]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:47:49.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:47:49 smithi083 ceph-mon[29774]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:47:49.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:47:49 smithi189 ceph-mon[25797]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:47:51.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:47:51 smithi083 ceph-mon[24810]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:51.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:47:51 smithi083 ceph-mon[29774]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:51.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:47:51 smithi189 ceph-mon[25797]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:51.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:47:51 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:47:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:47:51.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:47:51 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:47:51.645Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:47:51.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:47:51 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:47:51.646Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:47:51.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:47:51 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:47:51.647Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:47:53.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:47:53 smithi083 ceph-mon[24810]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:53.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:47:53 smithi083 ceph-mon[29774]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:53.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:47:53 smithi189 ceph-mon[25797]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:53.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:47:53 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:47:53.529Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:47:53.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:47:53 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:47:53.530Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:47:53.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:47:53 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:47:53.531Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:47:54.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:47:54 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:47:54.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:47:54 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:47:54.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:47:54 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:47:55.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:47:55 smithi083 ceph-mon[24810]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:47:55.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:47:55 smithi083 ceph-mon[29774]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:47:55.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:47:55 smithi189 ceph-mon[25797]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:47:56.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:47:56 smithi189 ceph-mon[25797]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:56.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:47:56 smithi083 ceph-mon[24810]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:56.675 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:47:56 smithi083 ceph-mon[29774]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:47:59.378 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:47:59 smithi083 ceph-mon[24810]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:47:59.379 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:47:59 smithi083 ceph-mon[29774]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:47:59.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:47:59 smithi189 ceph-mon[25797]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:48:01.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:48:01 smithi083 ceph-mon[24810]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:01.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:48:01 smithi083 ceph-mon[29774]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:01.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:48:01 smithi189 ceph-mon[25797]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:01.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:48:01 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:48:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:48:01.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:48:01 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:48:01.646Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:48:01.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:48:01 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:48:01.646Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:48:01.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:48:01 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:48:01.647Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:48:03.274 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:48:03 smithi189 ceph-mon[25797]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:03.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:48:03 smithi083 ceph-mon[24810]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:03.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:48:03 smithi083 ceph-mon[29774]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:03.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:48:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:48:03.530Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:48:03.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:48:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:48:03.531Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:48:03.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:48:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:48:03.532Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:48:04.413 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:48:04 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:48:04.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:48:04 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:48:04.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:48:04 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:48:05.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:48:05 smithi189 ceph-mon[25797]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:48:05.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:48:05 smithi083 ceph-mon[24810]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:48:05.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:48:05 smithi083 ceph-mon[29774]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:48:06.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:48:06 smithi083 ceph-mon[24810]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:06.675 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:48:06 smithi083 ceph-mon[29774]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:06.707 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:48:06 smithi189 ceph-mon[25797]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:09.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:48:09 smithi083 ceph-mon[24810]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:48:09.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:48:09 smithi083 ceph-mon[29774]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:48:09.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:48:09 smithi189 ceph-mon[25797]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:48:11.426 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:48:11 smithi083 ceph-mon[24810]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:11.427 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:48:11 smithi083 ceph-mon[29774]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:11.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:48:11 smithi189 ceph-mon[25797]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:11.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:48:11 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:48:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:48:11.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:48:11 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:48:11.646Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:48:11.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:48:11 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:48:11.647Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:48:11.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:48:11 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:48:11.648Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:48:13.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:48:13 smithi083 ceph-mon[24810]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:13.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:48:13 smithi083 ceph-mon[29774]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:13.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:48:13 smithi189 ceph-mon[25797]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:13.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:48:13 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:48:13.531Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:48:13.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:48:13 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:48:13.532Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:48:13.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:48:13 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:48:13.533Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:48:14.207 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:48:14 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:48:14.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:48:14 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:48:14.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:48:14 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:48:15.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:48:15 smithi083 ceph-mon[24810]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:48:15.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:48:15 smithi083 ceph-mon[29774]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:48:15.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:48:15 smithi189 ceph-mon[25797]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:48:16.639 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:48:16 smithi083 ceph-mon[24810]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:16.639 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:48:16 smithi083 ceph-mon[29774]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:16.684 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:48:16 smithi189 ceph-mon[25797]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:19.154 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:48:19 smithi189 ceph-mon[25797]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:48:19.334 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:48:19 smithi083 ceph-mon[24810]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:48:19.334 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:48:19 smithi083 ceph-mon[29774]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:48:21.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:48:21 smithi083 ceph-mon[24810]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:21.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:48:21 smithi083 ceph-mon[29774]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:21.423 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:48:21 smithi189 ceph-mon[25797]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:21.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:48:21 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:48:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:48:21.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:48:21 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:48:21.647Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:48:21.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:48:21 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:48:21.647Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:48:21.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:48:21 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:48:21.648Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:48:23.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:48:23 smithi083 ceph-mon[24810]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:23.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:48:23 smithi083 ceph-mon[29774]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:23.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:48:23 smithi189 ceph-mon[25797]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:23.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:48:23 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:48:23.532Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:48:23.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:48:23 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:48:23.533Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:48:23.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:48:23 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:48:23.535Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:48:24.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:48:24 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:48:24.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:48:24 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:48:24.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:48:24 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:48:25.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:48:25 smithi189 ceph-mon[25797]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:48:25.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:48:25 smithi083 ceph-mon[24810]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:48:25.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:48:25 smithi083 ceph-mon[29774]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:48:26.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:48:26 smithi083 ceph-mon[24810]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:26.675 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:48:26 smithi083 ceph-mon[29774]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:26.707 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:48:26 smithi189 ceph-mon[25797]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:29.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:48:29 smithi083 ceph-mon[24810]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:48:29.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:48:29 smithi083 ceph-mon[29774]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:48:29.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:48:29 smithi189 ceph-mon[25797]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:48:31.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:48:31 smithi083 ceph-mon[24810]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:31.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:48:31 smithi083 ceph-mon[29774]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:31.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:48:31 smithi189 ceph-mon[25797]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:31.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:48:31 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:48:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:48:31.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:48:31 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:48:31.648Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:48:31.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:48:31 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:48:31.648Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:48:31.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:48:31 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:48:31.649Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:48:33.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:48:33 smithi083 ceph-mon[24810]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:33.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:48:33 smithi083 ceph-mon[29774]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:33.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:48:33 smithi189 ceph-mon[25797]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:33.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:48:33 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:48:33.532Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:48:33.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:48:33 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:48:33.533Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:48:33.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:48:33 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:48:33.534Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:48:34.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:48:34 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:48:34.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:48:34 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:48:34.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:48:34 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:48:35.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:48:35 smithi083 ceph-mon[24810]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:48:35.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:48:35 smithi083 ceph-mon[29774]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:48:35.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:48:35 smithi189 ceph-mon[25797]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:48:36.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:48:36 smithi083 ceph-mon[24810]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:36.675 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:48:36 smithi083 ceph-mon[29774]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:36.707 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:48:36 smithi189 ceph-mon[25797]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:39.075 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:48:39 smithi083 ceph-mon[24810]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:48:39.075 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:48:39 smithi083 ceph-mon[29774]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:48:39.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:48:39 smithi189 ceph-mon[25797]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:48:41.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:48:41 smithi083 ceph-mon[24810]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:41.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:48:41 smithi083 ceph-mon[29774]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:41.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:48:41 smithi189 ceph-mon[25797]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:41.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:48:41 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:48:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:48:41.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:48:41 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:48:41.649Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:48:41.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:48:41 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:48:41.650Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:48:41.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:48:41 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:48:41.651Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:48:43.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:48:43 smithi083 ceph-mon[24810]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:43.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:48:43 smithi083 ceph-mon[29774]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:43.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:48:43 smithi189 ceph-mon[25797]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:43.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:48:43 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:48:43.533Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:48:43.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:48:43 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:48:43.534Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:48:43.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:48:43 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:48:43.535Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:48:45.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:48:45 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:48:45.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:48:45 smithi083 ceph-mon[24810]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:48:45.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:48:45 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:48:45.426 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:48:45 smithi083 ceph-mon[29774]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:48:45.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:48:45 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:48:45.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:48:45 smithi189 ceph-mon[25797]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:48:46.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:48:46 smithi083 ceph-mon[24810]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:46.675 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:48:46 smithi083 ceph-mon[29774]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:46.707 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:48:46 smithi189 ceph-mon[25797]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:49.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:48:49 smithi083 ceph-mon[29774]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:48:49.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:48:49 smithi083 ceph-mon[24810]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:48:49.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:48:49 smithi189 ceph-mon[25797]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:48:51.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:48:51 smithi083 ceph-mon[24810]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:51.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:48:51 smithi083 ceph-mon[29774]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:51.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:48:51 smithi189 ceph-mon[25797]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:51.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:48:51 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:48:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:48:51.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:48:51 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:48:51.650Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:48:51.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:48:51 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:48:51.651Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:48:51.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:48:51 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:48:51.652Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:48:53.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:48:53 smithi083 ceph-mon[24810]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:53.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:48:53 smithi083 ceph-mon[29774]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:53.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:48:53 smithi189 ceph-mon[25797]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:53.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:48:53 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:48:53.533Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:48:53.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:48:53 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:48:53.534Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:48:53.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:48:53 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:48:53.535Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:48:55.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:48:55 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:48:55.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:48:55 smithi083 ceph-mon[24810]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:48:55.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:48:55 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:48:55.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:48:55 smithi083 ceph-mon[29774]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:48:55.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:48:55 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:48:55.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:48:55 smithi189 ceph-mon[25797]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-07T14:48:56.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:48:56 smithi083 ceph-mon[24810]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:56.676 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:48:56 smithi083 ceph-mon[29774]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:56.707 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:48:56 smithi189 ceph-mon[25797]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 68 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-07T14:48:59.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:48:59 smithi083 ceph-mon[24810]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 77 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-04-07T14:48:59.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:48:59 smithi083 ceph-mon[29774]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 77 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-04-07T14:48:59.207 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:48:59 smithi189 ceph-mon[25797]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 77 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-04-07T14:49:01.175 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:49:01 smithi083 ceph-mon[24810]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 77 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-04-07T14:49:01.175 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:49:01 smithi083 ceph-mon[29774]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 77 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-04-07T14:49:01.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:49:01 smithi189 ceph-mon[25797]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 77 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-04-07T14:49:01.925 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:49:01 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y[25045]: ::ffff:172.21.15.189 - - [07/Apr/2024:14:49:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-07T14:49:01.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:49:01 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:49:01.651Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:49:01.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:49:01 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:49:01.651Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:49:01.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:49:01 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:49:01.653Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:49:03.425 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:49:03 smithi083 ceph-mon[29774]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 77 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-04-07T14:49:03.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:49:03 smithi083 ceph-mon[24810]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 77 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-04-07T14:49:03.457 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:49:03 smithi189 ceph-mon[25797]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 77 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 0 B/s wr, 60 op/s 2024-04-07T14:49:03.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:49:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:49:03.534Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:49:03.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:49:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:49:03.535Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:49:03.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:49:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:49:03.536Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:49:05.038 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -e sha1=e5c885fe95ee982ff7b45214daa61b1a6d09d57d -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-07T14:49:05.354 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:49:05 smithi083 ceph-mon[24810]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:49:05.354 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:49:05 smithi083 ceph-mon[24810]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 89 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-07T14:49:05.354 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:49:05 smithi083 ceph-mon[29774]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:49:05.354 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:49:05 smithi083 ceph-mon[29774]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 89 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-07T14:49:05.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:49:05 smithi189 ceph-mon[25797]: from='client.24763 -' entity='client.iscsi.foo.smithi083.qmxyme' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-07T14:49:05.458 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:49:05 smithi189 ceph-mon[25797]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 89 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-07T14:49:06.490 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:49:06 smithi083 ceph-mon[24810]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 89 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-07T14:49:06.490 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:49:06 smithi083 ceph-mon[29774]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 89 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-07T14:49:06.707 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:49:06 smithi189 ceph-mon[25797]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 89 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-07T14:49:07.370 INFO:teuthology.orchestra.run.smithi083.stderr:no valid command found; 10 closest matches: 2024-04-07T14:49:07.370 INFO:teuthology.orchestra.run.smithi083.stderr:pg stat 2024-04-07T14:49:07.370 INFO:teuthology.orchestra.run.smithi083.stderr:pg getmap 2024-04-07T14:49:07.370 INFO:teuthology.orchestra.run.smithi083.stderr:pg dump [...] 2024-04-07T14:49:07.370 INFO:teuthology.orchestra.run.smithi083.stderr:pg dump_json [...] 2024-04-07T14:49:07.370 INFO:teuthology.orchestra.run.smithi083.stderr:pg dump_pools_json 2024-04-07T14:49:07.370 INFO:teuthology.orchestra.run.smithi083.stderr:pg ls-by-pool [...] 2024-04-07T14:49:07.370 INFO:teuthology.orchestra.run.smithi083.stderr:pg ls-by-primary [] [...] 2024-04-07T14:49:07.371 INFO:teuthology.orchestra.run.smithi083.stderr:pg ls-by-osd [] [...] 2024-04-07T14:49:07.371 INFO:teuthology.orchestra.run.smithi083.stderr:pg ls [] [...] 2024-04-07T14:49:07.371 INFO:teuthology.orchestra.run.smithi083.stderr:pg dump_stuck [...] [] 2024-04-07T14:49:07.371 INFO:teuthology.orchestra.run.smithi083.stderr:Error EINVAL: invalid command 2024-04-07T14:49:07.675 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:49:07 smithi083 ceph-mon[24810]: from='client.? 172.21.15.83:0/400897385' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-07T14:49:07.675 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:49:07 smithi083 ceph-mon[29774]: from='client.? 172.21.15.83:0/400897385' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-07T14:49:07.707 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:49:07 smithi189 ceph-mon[25797]: from='client.? 172.21.15.83:0/400897385' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-07T14:49:07.998 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-04-07T14:49:07.999 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_e5c885fe95ee982ff7b45214daa61b1a6d09d57d/qa/tasks/cephadm.py", line 1459, in shell _shell( File "/home/teuthworker/src/git.ceph.com_ceph_e5c885fe95ee982ff7b45214daa61b1a6d09d57d/qa/tasks/cephadm.py", line 110, in _shell return remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi083 with status 22: 'sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -e sha1=e5c885fe95ee982ff7b45214daa61b1a6d09d57d -- bash -c \'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1\'' 2024-04-07T14:49:08.205 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=1dd0df8d3d5f4dacbd8eaf9265a7df31 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_e5c885fe95ee982ff7b45214daa61b1a6d09d57d/qa/tasks/cephadm.py", line 1459, in shell _shell( File "/home/teuthworker/src/git.ceph.com_ceph_e5c885fe95ee982ff7b45214daa61b1a6d09d57d/qa/tasks/cephadm.py", line 110, in _shell return remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi083 with status 22: 'sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -e sha1=e5c885fe95ee982ff7b45214daa61b1a6d09d57d -- bash -c \'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1\'' 2024-04-07T14:49:08.208 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-04-07T14:49:08.218 INFO:tasks.cephadm:Teardown begin 2024-04-07T14:49:08.219 DEBUG:teuthology.orchestra.run.smithi083:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-07T14:49:08.257 DEBUG:teuthology.orchestra.run.smithi189:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-07T14:49:08.297 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-04-07T14:49:08.297 DEBUG:teuthology.orchestra.run.smithi083:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-07T14:49:08.314 DEBUG:teuthology.orchestra.run.smithi189:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-07T14:49:08.353 INFO:tasks.cephadm:Stopping all daemons... 2024-04-07T14:49:08.354 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-04-07T14:49:08.354 DEBUG:teuthology.orchestra.run.smithi083:> sudo systemctl stop ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@mon.a 2024-04-07T14:49:08.545 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:49:08 smithi083 ceph-mon[24810]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 89 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-07T14:49:08.545 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:49:08 smithi083 systemd[1]: Stopping Ceph mon.a for 7388281a-f4eb-11ee-b649-cb9ed24678a4... 2024-04-07T14:49:08.545 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:49:08 smithi083 bash[65322]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mon.a" found: no such container 2024-04-07T14:49:08.546 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:49:08 smithi083 ceph-mon[29774]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 89 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-07T14:49:08.707 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:49:08 smithi189 ceph-mon[25797]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 89 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-07T14:49:08.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:49:08 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mon-a[24806]: 2024-04-07T14:49:08.545+0000 7f848f890700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-07T14:49:08.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:49:08 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mon-a[24806]: 2024-04-07T14:49:08.545+0000 7f848f890700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-04-07T14:49:08.925 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:49:08 smithi083 podman[65335]: 2024-04-07 14:49:08.761090644 +0000 UTC m=+0.321710763 container died 045b78f11b8e0a839a0de89ffde1b01200a6e2b3f8070084bb844c2a9e030bc8 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mon-a, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, version=8, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, maintainer=Guillaume Abrioux , name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, release=754, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, ceph=True, vendor=Red Hat, Inc.) 2024-04-07T14:49:09.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:49:09 smithi083 podman[65335]: 2024-04-07 14:49:09.137775708 +0000 UTC m=+0.698395829 container cleanup 045b78f11b8e0a839a0de89ffde1b01200a6e2b3f8070084bb844c2a9e030bc8 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mon-a, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, version=8, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, name=centos-stream, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-04-07T14:49:09.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:49:09 smithi083 bash[65335]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mon-a 2024-04-07T14:49:09.425 INFO:journalctl@ceph.mon.a.smithi083.stdout:Apr 07 14:49:09 smithi083 bash[65363]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mon.a" found: no such container 2024-04-07T14:49:09.443 DEBUG:teuthology.orchestra.run.smithi083:> sudo pkill -f 'journalctl -f -n 0 -u ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@mon.a.service' 2024-04-07T14:49:09.502 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-07T14:49:09.503 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-04-07T14:49:09.503 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-04-07T14:49:09.503 DEBUG:teuthology.orchestra.run.smithi083:> sudo systemctl stop ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@mon.c 2024-04-07T14:49:09.783 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:49:09 smithi083 systemd[1]: Stopping Ceph mon.c for 7388281a-f4eb-11ee-b649-cb9ed24678a4... 2024-04-07T14:49:09.783 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:49:09 smithi083 bash[65417]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mon.c" found: no such container 2024-04-07T14:49:09.784 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:49:09 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mon-c[29758]: 2024-04-07T14:49:09.683+0000 7fe320b9b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-07T14:49:09.784 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:49:09 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mon-c[29758]: 2024-04-07T14:49:09.683+0000 7fe320b9b700 -1 mon.c@1(peon) e3 *** Got Signal Terminated *** 2024-04-07T14:49:10.065 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:49:09 smithi083 podman[65430]: 2024-04-07 14:49:09.785841507 +0000 UTC m=+0.200227580 container died 69e62d2b9b32fe95f2e1bf6a4fb1865ea5a76d5c270a33004c109b11806c291c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mon-c, io.buildah.version=1.19.8, io.openshift.expose-services=, name=centos-stream, release=754, RELEASE=HEAD, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, vcs-type=git, com.redhat.component=centos-stream-container, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, maintainer=Guillaume Abrioux ) 2024-04-07T14:49:10.324 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:49:10 smithi083 podman[65430]: 2024-04-07 14:49:10.142259974 +0000 UTC m=+0.556646057 container cleanup 69e62d2b9b32fe95f2e1bf6a4fb1865ea5a76d5c270a33004c109b11806c291c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mon-c, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, name=centos-stream, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, release=754, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container) 2024-04-07T14:49:10.324 INFO:journalctl@ceph.mon.c.smithi083.stdout:Apr 07 14:49:10 smithi083 bash[65430]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mon-c 2024-04-07T14:49:10.368 DEBUG:teuthology.orchestra.run.smithi083:> sudo pkill -f 'journalctl -f -n 0 -u ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@mon.c.service' 2024-04-07T14:49:10.410 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-07T14:49:10.411 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-04-07T14:49:10.411 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-04-07T14:49:10.411 DEBUG:teuthology.orchestra.run.smithi189:> sudo systemctl stop ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@mon.b 2024-04-07T14:49:10.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:49:10 smithi189 systemd[1]: Stopping Ceph mon.b for 7388281a-f4eb-11ee-b649-cb9ed24678a4... 2024-04-07T14:49:10.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:49:10 smithi189 bash[55193]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mon.b" found: no such container 2024-04-07T14:49:10.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:49:10 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mon-b[25789]: 2024-04-07T14:49:10.586+0000 7fd851918700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-07T14:49:10.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:49:10 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mon-b[25789]: 2024-04-07T14:49:10.586+0000 7fd851918700 -1 mon.b@2(peon) e3 *** Got Signal Terminated *** 2024-04-07T14:49:10.708 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:49:10 smithi189 podman[55206]: 2024-04-07 14:49:10.659097168 +0000 UTC m=+0.161141274 container died 76a325b30382a4989cb99af8779dff1473e54bc0614cf0362ad1569f25e73405 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mon-b, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, architecture=x86_64, distribution-scope=public, release=754, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., RELEASE=HEAD, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, GIT_CLEAN=True) 2024-04-07T14:49:11.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:49:11 smithi189 podman[55206]: 2024-04-07 14:49:11.121014868 +0000 UTC m=+0.623058965 container cleanup 76a325b30382a4989cb99af8779dff1473e54bc0614cf0362ad1569f25e73405 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mon-b, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, distribution-scope=public, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, architecture=x86_64, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, release=754, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-07T14:49:11.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:49:11 smithi189 bash[55206]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mon-b 2024-04-07T14:49:11.208 INFO:journalctl@ceph.mon.b.smithi189.stdout:Apr 07 14:49:11 smithi189 bash[55232]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mon.b" found: no such container 2024-04-07T14:49:11.418 DEBUG:teuthology.orchestra.run.smithi189:> sudo pkill -f 'journalctl -f -n 0 -u ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@mon.b.service' 2024-04-07T14:49:11.473 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-07T14:49:11.473 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-04-07T14:49:11.473 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-04-07T14:49:11.474 DEBUG:teuthology.orchestra.run.smithi083:> sudo systemctl stop ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@mgr.y 2024-04-07T14:49:11.926 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:49:11 smithi083 systemd[1]: Stopping Ceph mgr.y for 7388281a-f4eb-11ee-b649-cb9ed24678a4... 2024-04-07T14:49:11.926 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:49:11 smithi083 bash[65510]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr.y" found: no such container 2024-04-07T14:49:11.926 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:49:11 smithi083 podman[65522]: 2024-04-07 14:49:11.702794675 +0000 UTC m=+0.140860857 container died 966a9d7367c02082435a2fe3f0164915fcc0829a390f6cde0cc8118c38bae02b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y, RELEASE=HEAD, maintainer=Guillaume Abrioux , release=754, version=8, io.openshift.expose-services=, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, name=centos-stream, vcs-type=git, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, ceph=True, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-07T14:49:11.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:49:11 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:49:11.652Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:49:11.927 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:49:11 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:49:11.653Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:49:11.927 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:49:11 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:49:11.654Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.83 because it doesn't contain any IP SANs" 2024-04-07T14:49:12.363 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:49:12 smithi083 podman[65522]: 2024-04-07 14:49:12.063833307 +0000 UTC m=+0.501899490 container cleanup 966a9d7367c02082435a2fe3f0164915fcc0829a390f6cde0cc8118c38bae02b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, version=8, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-07T14:49:12.363 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:49:12 smithi083 bash[65522]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-y 2024-04-07T14:49:12.363 INFO:journalctl@ceph.mgr.y.smithi083.stdout:Apr 07 14:49:12 smithi083 bash[65547]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr.y" found: no such container 2024-04-07T14:49:12.363 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:49:12 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:49:12.145Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=2 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": dial tcp 172.21.15.83:8443: connect: connection refused" 2024-04-07T14:49:12.368 DEBUG:teuthology.orchestra.run.smithi083:> sudo pkill -f 'journalctl -f -n 0 -u ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@mgr.y.service' 2024-04-07T14:49:12.406 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-07T14:49:12.406 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-04-07T14:49:12.406 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-04-07T14:49:12.406 DEBUG:teuthology.orchestra.run.smithi189:> sudo systemctl stop ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@mgr.x 2024-04-07T14:49:12.708 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:49:12 smithi189 systemd[1]: Stopping Ceph mgr.x for 7388281a-f4eb-11ee-b649-cb9ed24678a4... 2024-04-07T14:49:12.708 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:49:12 smithi189 bash[55286]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr.x" found: no such container 2024-04-07T14:49:12.708 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:49:12 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x[50602]: 2024-04-07T14:49:12.609+0000 7ff29243d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.x -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-07T14:49:12.708 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:49:12 smithi189 podman[55298]: 2024-04-07 14:49:12.619998079 +0000 UTC m=+0.128239272 container died 0ef726e5100dc85819cb69a35bea77af315f205e88937c66865b6561d5b5f195 (image=quay.ceph.io/ceph-ci/ceph:e5c885fe95ee982ff7b45214daa61b1a6d09d57d, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=main-e5c885f, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326) 2024-04-07T14:49:13.158 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:49:12 smithi189 podman[55298]: 2024-04-07 14:49:12.959245744 +0000 UTC m=+0.467486942 container cleanup 0ef726e5100dc85819cb69a35bea77af315f205e88937c66865b6561d5b5f195 (image=quay.ceph.io/ceph-ci/ceph:e5c885fe95ee982ff7b45214daa61b1a6d09d57d, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, RELEASE=main-e5c885f, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=) 2024-04-07T14:49:13.159 INFO:journalctl@ceph.mgr.x.smithi189.stdout:Apr 07 14:49:12 smithi189 bash[55298]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-mgr-x 2024-04-07T14:49:13.195 DEBUG:teuthology.orchestra.run.smithi189:> sudo pkill -f 'journalctl -f -n 0 -u ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@mgr.x.service' 2024-04-07T14:49:13.235 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-07T14:49:13.235 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-04-07T14:49:13.236 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-04-07T14:49:13.236 DEBUG:teuthology.orchestra.run.smithi083:> sudo systemctl stop ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@osd.0 2024-04-07T14:49:13.532 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:49:13 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:49:13.479Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=7 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": dial tcp 172.21.15.83:8443: connect: connection refused" 2024-04-07T14:49:13.532 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 07 14:49:13 smithi083 systemd[1]: Stopping Ceph osd.0 for 7388281a-f4eb-11ee-b649-cb9ed24678a4... 2024-04-07T14:49:13.532 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 07 14:49:13 smithi083 bash[65603]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd.0" found: no such container 2024-04-07T14:49:13.533 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 07 14:49:13 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-0[32973]: 2024-04-07T14:49:13.450+0000 7f914dbef700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-07T14:49:13.533 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 07 14:49:13 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-0[32973]: 2024-04-07T14:49:13.450+0000 7f914dbef700 -1 osd.0 97 *** Got signal Terminated *** 2024-04-07T14:49:13.533 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 07 14:49:13 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-0[32973]: 2024-04-07T14:49:13.450+0000 7f914dbef700 -1 osd.0 97 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-07T14:49:13.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:49:13 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:49:13.534Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": dial tcp 172.21.15.83:8443: connect: connection refused" 2024-04-07T14:49:13.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:49:13 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:49:13.535Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": dial tcp 172.21.15.83:8443: connect: connection refused" 2024-04-07T14:49:13.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:49:13 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:49:13.535Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:49:19.275 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 07 14:49:18 smithi083 podman[65613]: 2024-04-07 14:49:18.973603839 +0000 UTC m=+5.658487011 container died 8e0c47a72b20317b9259e75a4ba5c5dbe187522a0712f553294511ba43d18767 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-0, ceph=True, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, version=8, maintainer=Guillaume Abrioux , architecture=x86_64, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-07T14:49:19.675 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 07 14:49:19 smithi083 podman[65613]: 2024-04-07 14:49:19.467789823 +0000 UTC m=+6.152672994 container cleanup 8e0c47a72b20317b9259e75a4ba5c5dbe187522a0712f553294511ba43d18767 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, version=8, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, architecture=x86_64, RELEASE=HEAD, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-07T14:49:19.675 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 07 14:49:19 smithi083 bash[65613]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-0 2024-04-07T14:49:19.675 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 07 14:49:19 smithi083 bash[65664]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd.0" found: no such container 2024-04-07T14:49:19.947 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 07 14:49:19 smithi083 podman[65653]: 2024-04-07 14:49:19.685820096 +0000 UTC m=+0.710073087 container remove 8e0c47a72b20317b9259e75a4ba5c5dbe187522a0712f553294511ba43d18767 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-0, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.buildah.version=1.19.8, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, com.redhat.component=centos-stream-container, distribution-scope=public, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., name=centos-stream) 2024-04-07T14:49:21.425 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 07 14:49:21 smithi083 podman[65782]: 2024-04-07 14:49:21.163057812 +0000 UTC m=+0.267447002 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-07T14:49:21.771 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:49:21 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:49:21.653Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": dial tcp 172.21.15.83:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:49:21.772 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:49:21 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:49:21.654Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": dial tcp 172.21.15.83:8443: connect: connection refused" 2024-04-07T14:49:21.772 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:49:21 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:49:21.654Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:49:21.772 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 07 14:49:21 smithi083 podman[65782]: 2024-04-07 14:49:21.463393911 +0000 UTC m=+0.567783097 container create 0cd94f0b8836479df0a2aa5fe09c3d915246dc18c96cc1fe6fa5dcd1202891c8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-0-deactivate, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, architecture=x86_64, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, vcs-type=git, name=centos-stream, GIT_BRANCH=HEAD, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0) 2024-04-07T14:49:21.772 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 07 14:49:21 smithi083 podman[65782]: 2024-04-07 14:49:21.698913177 +0000 UTC m=+0.803302367 container init 0cd94f0b8836479df0a2aa5fe09c3d915246dc18c96cc1fe6fa5dcd1202891c8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-0-deactivate, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, version=8, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, io.openshift.expose-services=, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, vcs-type=git, GIT_CLEAN=True, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0) 2024-04-07T14:49:21.773 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 07 14:49:21 smithi083 podman[65782]: 2024-04-07 14:49:21.702086694 +0000 UTC m=+0.806475889 container start 0cd94f0b8836479df0a2aa5fe09c3d915246dc18c96cc1fe6fa5dcd1202891c8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-0-deactivate, io.openshift.expose-services=, vcs-type=git, architecture=x86_64, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754) 2024-04-07T14:49:21.773 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 07 14:49:21 smithi083 podman[65782]: 2024-04-07 14:49:21.774106149 +0000 UTC m=+0.878495344 container attach 0cd94f0b8836479df0a2aa5fe09c3d915246dc18c96cc1fe6fa5dcd1202891c8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-0-deactivate, ceph=True, version=8, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, GIT_CLEAN=True, name=centos-stream, io.buildah.version=1.19.8, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git) 2024-04-07T14:49:22.040 INFO:journalctl@ceph.osd.0.smithi083.stdout:Apr 07 14:49:22 smithi083 podman[65782]: 2024-04-07 14:49:22.018206448 +0000 UTC m=+1.122595643 container died 0cd94f0b8836479df0a2aa5fe09c3d915246dc18c96cc1fe6fa5dcd1202891c8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-0-deactivate, CEPH_POINT_RELEASE=-17.2.0, ceph=True, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, vendor=Red Hat, Inc., release=754) 2024-04-07T14:49:22.634 DEBUG:teuthology.orchestra.run.smithi083:> sudo pkill -f 'journalctl -f -n 0 -u ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@osd.0.service' 2024-04-07T14:49:22.676 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-07T14:49:22.676 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-04-07T14:49:22.677 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-04-07T14:49:22.677 DEBUG:teuthology.orchestra.run.smithi083:> sudo systemctl stop ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@osd.1 2024-04-07T14:49:22.883 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 07 14:49:22 smithi083 systemd[1]: Stopping Ceph osd.1 for 7388281a-f4eb-11ee-b649-cb9ed24678a4... 2024-04-07T14:49:23.175 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 07 14:49:22 smithi083 bash[66006]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd.1" found: no such container 2024-04-07T14:49:23.675 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 07 14:49:23 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-1[35861]: 2024-04-07T14:49:23.434+0000 7fa5a0f7d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-07T14:49:23.675 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 07 14:49:23 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-1[35861]: 2024-04-07T14:49:23.434+0000 7fa5a0f7d700 -1 osd.1 97 *** Got signal Terminated *** 2024-04-07T14:49:23.675 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 07 14:49:23 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-1[35861]: 2024-04-07T14:49:23.434+0000 7fa5a0f7d700 -1 osd.1 97 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-07T14:49:23.676 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:49:23 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:49:23.534Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": dial tcp 172.21.15.83:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:49:23.676 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:49:23 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:49:23.535Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": dial tcp 172.21.15.83:8443: connect: connection refused" 2024-04-07T14:49:23.676 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:49:23 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:49:23.535Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:49:29.121 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 07 14:49:28 smithi083 podman[66020]: 2024-04-07 14:49:28.823029335 +0000 UTC m=+5.929339070 container died 47f33a1b3c4945e45b49e634d00d9a68a2c89b5fca4af970cf96d0bf073e1420 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-1, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, release=754, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, vendor=Red Hat, Inc., version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, architecture=x86_64, vcs-type=git, name=centos-stream, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8) 2024-04-07T14:49:29.425 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 07 14:49:29 smithi083 podman[66020]: 2024-04-07 14:49:29.214059613 +0000 UTC m=+6.320369379 container cleanup 47f33a1b3c4945e45b49e634d00d9a68a2c89b5fca4af970cf96d0bf073e1420 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-1, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, io.buildah.version=1.19.8, GIT_CLEAN=True, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, distribution-scope=public, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-07T14:49:29.425 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 07 14:49:29 smithi083 bash[66020]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-1 2024-04-07T14:49:29.425 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 07 14:49:29 smithi083 bash[66568]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd.1" found: no such container 2024-04-07T14:49:29.738 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 07 14:49:29 smithi083 podman[66555]: 2024-04-07 14:49:29.490326763 +0000 UTC m=+0.666546993 container remove 47f33a1b3c4945e45b49e634d00d9a68a2c89b5fca4af970cf96d0bf073e1420 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-1, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, distribution-scope=public, io.buildah.version=1.19.8, vendor=Red Hat, Inc., ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git) 2024-04-07T14:49:29.739 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 07 14:49:29 smithi083 podman[66633]: 2024-04-07 14:49:29.60952678 +0000 UTC m=+0.019113506 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-07T14:49:29.739 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 07 14:49:29 smithi083 podman[66633]: 2024-04-07 14:49:29.741030182 +0000 UTC m=+0.150616900 container create 0a1f59837daae045fd4eec9fd34990ad912216e33fa7747adfabe2e0ccb3bd8b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-1-deactivate, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, version=8, distribution-scope=public, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-04-07T14:49:30.057 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 07 14:49:29 smithi083 podman[66633]: 2024-04-07 14:49:29.976761271 +0000 UTC m=+0.386347988 container init 0a1f59837daae045fd4eec9fd34990ad912216e33fa7747adfabe2e0ccb3bd8b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-1-deactivate, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, build-date=2022-05-03T08:36:31.336870, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, distribution-scope=public, name=centos-stream, GIT_CLEAN=True) 2024-04-07T14:49:30.058 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 07 14:49:29 smithi083 podman[66633]: 2024-04-07 14:49:29.979930792 +0000 UTC m=+0.389517516 container start 0a1f59837daae045fd4eec9fd34990ad912216e33fa7747adfabe2e0ccb3bd8b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-1-deactivate, RELEASE=HEAD, io.openshift.expose-services=, version=8, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, ceph=True, vendor=Red Hat, Inc.) 2024-04-07T14:49:30.058 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 07 14:49:30 smithi083 podman[66633]: 2024-04-07 14:49:30.060002524 +0000 UTC m=+0.469589255 container attach 0a1f59837daae045fd4eec9fd34990ad912216e33fa7747adfabe2e0ccb3bd8b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-1-deactivate, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, name=centos-stream, io.openshift.expose-services=, architecture=x86_64, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., ceph=True, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, GIT_CLEAN=True, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, maintainer=Guillaume Abrioux , vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD) 2024-04-07T14:49:30.425 INFO:journalctl@ceph.osd.1.smithi083.stdout:Apr 07 14:49:30 smithi083 podman[66633]: 2024-04-07 14:49:30.263317775 +0000 UTC m=+0.672904499 container died 0a1f59837daae045fd4eec9fd34990ad912216e33fa7747adfabe2e0ccb3bd8b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-1-deactivate, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, RELEASE=HEAD, architecture=x86_64, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, version=8, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, name=centos-stream, io.openshift.expose-services=, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-07T14:49:30.964 DEBUG:teuthology.orchestra.run.smithi083:> sudo pkill -f 'journalctl -f -n 0 -u ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@osd.1.service' 2024-04-07T14:49:31.013 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-07T14:49:31.013 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-04-07T14:49:31.014 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-04-07T14:49:31.014 DEBUG:teuthology.orchestra.run.smithi083:> sudo systemctl stop ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@osd.2 2024-04-07T14:49:31.206 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 07 14:49:31 smithi083 systemd[1]: Stopping Ceph osd.2 for 7388281a-f4eb-11ee-b649-cb9ed24678a4... 2024-04-07T14:49:31.207 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 07 14:49:31 smithi083 bash[66712]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd.2" found: no such container 2024-04-07T14:49:31.652 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 07 14:49:31 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-2[38815]: 2024-04-07T14:49:31.208+0000 7f305ade5700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-07T14:49:31.652 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 07 14:49:31 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-2[38815]: 2024-04-07T14:49:31.208+0000 7f305ade5700 -1 osd.2 97 *** Got signal Terminated *** 2024-04-07T14:49:31.652 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 07 14:49:31 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-2[38815]: 2024-04-07T14:49:31.208+0000 7f305ade5700 -1 osd.2 97 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-07T14:49:31.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:49:31 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:49:31.654Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": dial tcp 172.21.15.83:8443: connect: connection refused" 2024-04-07T14:49:31.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:49:31 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:49:31.654Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": dial tcp 172.21.15.83:8443: connect: connection refused" 2024-04-07T14:49:31.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:49:31 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:49:31.654Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:49:33.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:49:33 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:49:33.535Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": dial tcp 172.21.15.83:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:49:33.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:49:33 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:49:33.536Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": dial tcp 172.21.15.83:8443: connect: connection refused" 2024-04-07T14:49:33.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:49:33 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:49:33.536Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:49:36.815 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 07 14:49:36 smithi083 podman[66724]: 2024-04-07 14:49:36.527823003 +0000 UTC m=+5.434994538 container died 0747abc2af4a84cc47df10a2b08c08eab9842fe09841ec5d4780ff805f1f82bd (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-2, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, distribution-scope=public, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, ceph=True, com.redhat.component=centos-stream-container, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, io.openshift.expose-services=, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754) 2024-04-07T14:49:37.175 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 07 14:49:37 smithi083 podman[66724]: 2024-04-07 14:49:37.017170899 +0000 UTC m=+5.924342432 container cleanup 0747abc2af4a84cc47df10a2b08c08eab9842fe09841ec5d4780ff805f1f82bd (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-2, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, architecture=x86_64, ceph=True, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, GIT_BRANCH=HEAD, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container) 2024-04-07T14:49:37.175 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 07 14:49:37 smithi083 bash[66724]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-2 2024-04-07T14:49:37.175 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 07 14:49:37 smithi083 bash[66765]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd.2" found: no such container 2024-04-07T14:49:37.633 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 07 14:49:37 smithi083 podman[66752]: 2024-04-07 14:49:37.294041028 +0000 UTC m=+0.757252002 container remove 0747abc2af4a84cc47df10a2b08c08eab9842fe09841ec5d4780ff805f1f82bd (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-2, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, architecture=x86_64, distribution-scope=public, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, vcs-type=git, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, vendor=Red Hat, Inc., RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, version=8, io.openshift.expose-services=, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-07T14:49:37.633 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 07 14:49:37 smithi083 podman[66830]: 2024-04-07 14:49:37.397996704 +0000 UTC m=+0.018732829 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-07T14:49:37.633 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 07 14:49:37 smithi083 podman[66830]: 2024-04-07 14:49:37.528230428 +0000 UTC m=+0.148966587 container create 20c2dbabcea28a12bd6a404e27975ce1668f35f926e20e3175456aa2dde7fbc7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-2-deactivate, architecture=x86_64, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , version=8, ceph=True, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, com.redhat.component=centos-stream-container) 2024-04-07T14:49:37.926 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 07 14:49:37 smithi083 podman[66830]: 2024-04-07 14:49:37.772182193 +0000 UTC m=+0.392918311 container init 20c2dbabcea28a12bd6a404e27975ce1668f35f926e20e3175456aa2dde7fbc7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-2-deactivate, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, vcs-type=git, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, io.buildah.version=1.19.8, name=centos-stream, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , version=8, architecture=x86_64) 2024-04-07T14:49:37.926 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 07 14:49:37 smithi083 podman[66830]: 2024-04-07 14:49:37.77555473 +0000 UTC m=+0.396290850 container start 20c2dbabcea28a12bd6a404e27975ce1668f35f926e20e3175456aa2dde7fbc7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-2-deactivate, architecture=x86_64, GIT_BRANCH=HEAD, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., name=centos-stream, version=8, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, vcs-type=git, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, ceph=True, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0) 2024-04-07T14:49:37.927 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 07 14:49:37 smithi083 podman[66830]: 2024-04-07 14:49:37.863710204 +0000 UTC m=+0.484446327 container attach 20c2dbabcea28a12bd6a404e27975ce1668f35f926e20e3175456aa2dde7fbc7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-2-deactivate, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, com.redhat.component=centos-stream-container, GIT_CLEAN=True, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, distribution-scope=public, architecture=x86_64, release=754, vendor=Red Hat, Inc., version=8, vcs-type=git, RELEASE=HEAD) 2024-04-07T14:49:38.347 INFO:journalctl@ceph.osd.2.smithi083.stdout:Apr 07 14:49:38 smithi083 podman[66830]: 2024-04-07 14:49:38.057939123 +0000 UTC m=+0.678675230 container died 20c2dbabcea28a12bd6a404e27975ce1668f35f926e20e3175456aa2dde7fbc7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-2-deactivate, version=8, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, RELEASE=HEAD, io.buildah.version=1.19.8, io.openshift.expose-services=, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , distribution-scope=public, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-07T14:49:38.648 DEBUG:teuthology.orchestra.run.smithi083:> sudo pkill -f 'journalctl -f -n 0 -u ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@osd.2.service' 2024-04-07T14:49:38.689 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-07T14:49:38.689 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-04-07T14:49:38.690 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-04-07T14:49:38.690 DEBUG:teuthology.orchestra.run.smithi083:> sudo systemctl stop ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@osd.3 2024-04-07T14:49:38.909 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 07 14:49:38 smithi083 systemd[1]: Stopping Ceph osd.3 for 7388281a-f4eb-11ee-b649-cb9ed24678a4... 2024-04-07T14:49:38.910 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 07 14:49:38 smithi083 bash[66908]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd.3" found: no such container 2024-04-07T14:49:39.175 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 07 14:49:38 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-3[41825]: 2024-04-07T14:49:38.910+0000 7fe8fa534700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-07T14:49:39.175 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 07 14:49:38 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-3[41825]: 2024-04-07T14:49:38.910+0000 7fe8fa534700 -1 osd.3 97 *** Got signal Terminated *** 2024-04-07T14:49:39.175 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 07 14:49:38 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-3[41825]: 2024-04-07T14:49:38.910+0000 7fe8fa534700 -1 osd.3 97 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-07T14:49:40.675 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 07 14:49:40 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-3[41825]: 2024-04-07T14:49:40.620+0000 7fe8f66ce700 -1 osd.3 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:15.300067+0000 front 2024-04-07T14:49:15.300126+0000 (oldest deadline 2024-04-07T14:49:40.000175+0000) 2024-04-07T14:49:41.925 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 07 14:49:41 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-3[41825]: 2024-04-07T14:49:41.651+0000 7fe8f66ce700 -1 osd.3 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:15.300067+0000 front 2024-04-07T14:49:15.300126+0000 (oldest deadline 2024-04-07T14:49:40.000175+0000) 2024-04-07T14:49:41.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:49:41 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:49:41.655Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": dial tcp 172.21.15.83:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:49:41.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:49:41 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:49:41.655Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": dial tcp 172.21.15.83:8443: connect: connection refused" 2024-04-07T14:49:41.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:49:41 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:49:41.655Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:49:42.458 INFO:journalctl@ceph.osd.4.smithi189.stdout:Apr 07 14:49:42 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-4[30531]: 2024-04-07T14:49:42.145+0000 7f9536a9b700 -1 osd.4 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:18.122923+0000 front 2024-04-07T14:49:18.122987+0000 (oldest deadline 2024-04-07T14:49:41.622883+0000) 2024-04-07T14:49:42.835 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 07 14:49:42 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-3[41825]: 2024-04-07T14:49:42.609+0000 7fe8f66ce700 -1 osd.3 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:15.300067+0000 front 2024-04-07T14:49:15.300126+0000 (oldest deadline 2024-04-07T14:49:40.000175+0000) 2024-04-07T14:49:43.175 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:49:42 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:49:42.869+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:16.385849+0000 front 2024-04-07T14:49:16.385969+0000 (oldest deadline 2024-04-07T14:49:42.286011+0000) 2024-04-07T14:49:43.457 INFO:journalctl@ceph.osd.4.smithi189.stdout:Apr 07 14:49:43 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-4[30531]: 2024-04-07T14:49:43.176+0000 7f9536a9b700 -1 osd.4 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:18.122923+0000 front 2024-04-07T14:49:18.122987+0000 (oldest deadline 2024-04-07T14:49:41.622883+0000) 2024-04-07T14:49:43.925 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 07 14:49:43 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-3[41825]: 2024-04-07T14:49:43.616+0000 7fe8f66ce700 -1 osd.3 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:15.300067+0000 front 2024-04-07T14:49:15.300126+0000 (oldest deadline 2024-04-07T14:49:40.000175+0000) 2024-04-07T14:49:43.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:49:43 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:49:43.536Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": dial tcp 172.21.15.83:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:49:43.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:49:43 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:49:43.536Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": dial tcp 172.21.15.83:8443: connect: connection refused" 2024-04-07T14:49:43.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:49:43 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:49:43.537Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:49:44.137 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:49:43 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:49:43.859+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:16.385849+0000 front 2024-04-07T14:49:16.385969+0000 (oldest deadline 2024-04-07T14:49:42.286011+0000) 2024-04-07T14:49:44.138 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:49:44 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5[33437]: 2024-04-07T14:49:44.105+0000 7fd1945f7700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:18.211303+0000 front 2024-04-07T14:49:18.211329+0000 (oldest deadline 2024-04-07T14:49:43.511098+0000) 2024-04-07T14:49:44.459 INFO:journalctl@ceph.osd.4.smithi189.stdout:Apr 07 14:49:44 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-4[30531]: 2024-04-07T14:49:44.139+0000 7f9536a9b700 -1 osd.4 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:18.122923+0000 front 2024-04-07T14:49:18.122987+0000 (oldest deadline 2024-04-07T14:49:41.622883+0000) 2024-04-07T14:49:44.589 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 07 14:49:44 smithi083 podman[66918]: 2024-04-07 14:49:44.317794233 +0000 UTC m=+5.510729192 container died 57cbb5ed0f86ac6d9be63160cda105af54a205ac08a91ad0ce617d4f406c8dee (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-3, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, version=8, RELEASE=HEAD, architecture=x86_64, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, vendor=Red Hat, Inc., vcs-type=git, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, build-date=2022-05-03T08:36:31.336870, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD) 2024-04-07T14:49:44.925 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 07 14:49:44 smithi083 podman[66918]: 2024-04-07 14:49:44.682059794 +0000 UTC m=+5.874994793 container cleanup 57cbb5ed0f86ac6d9be63160cda105af54a205ac08a91ad0ce617d4f406c8dee (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-3, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, release=754, version=8, RELEASE=HEAD, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public) 2024-04-07T14:49:44.925 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 07 14:49:44 smithi083 bash[66918]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-3 2024-04-07T14:49:44.925 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 07 14:49:44 smithi083 bash[67057]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd.3" found: no such container 2024-04-07T14:49:45.084 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:49:44 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:49:44.905+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:16.385849+0000 front 2024-04-07T14:49:16.385969+0000 (oldest deadline 2024-04-07T14:49:42.286011+0000) 2024-04-07T14:49:45.085 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:49:44 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:49:44.802+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:12.776374+0000 front 2024-04-07T14:49:12.776258+0000 (oldest deadline 2024-04-07T14:49:44.576770+0000) 2024-04-07T14:49:45.299 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 07 14:49:44 smithi083 podman[67043]: 2024-04-07 14:49:44.958698729 +0000 UTC m=+0.642650734 container remove 57cbb5ed0f86ac6d9be63160cda105af54a205ac08a91ad0ce617d4f406c8dee (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-3, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, name=centos-stream, GIT_CLEAN=True, version=8, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, ceph=True, distribution-scope=public, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, vcs-type=git, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-07T14:49:45.299 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 07 14:49:45 smithi083 podman[67123]: 2024-04-07 14:49:45.039247623 +0000 UTC m=+0.019096865 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-07T14:49:45.299 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 07 14:49:45 smithi083 podman[67123]: 2024-04-07 14:49:45.192129283 +0000 UTC m=+0.171978514 container create bc9d1a1a82566d63bbe4bdf094bdee7f5f3555163a787f8b9309791ad075b4a5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-3-deactivate, release=754, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, maintainer=Guillaume Abrioux , vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, ceph=True, io.buildah.version=1.19.8) 2024-04-07T14:49:45.457 INFO:journalctl@ceph.osd.4.smithi189.stdout:Apr 07 14:49:45 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-4[30531]: 2024-04-07T14:49:45.158+0000 7f9536a9b700 -1 osd.4 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:18.122923+0000 front 2024-04-07T14:49:18.122987+0000 (oldest deadline 2024-04-07T14:49:41.622883+0000) 2024-04-07T14:49:45.458 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:49:45 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5[33437]: 2024-04-07T14:49:45.118+0000 7fd1945f7700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:18.211303+0000 front 2024-04-07T14:49:18.211329+0000 (oldest deadline 2024-04-07T14:49:43.511098+0000) 2024-04-07T14:49:45.597 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 07 14:49:45 smithi083 podman[67123]: 2024-04-07 14:49:45.419836554 +0000 UTC m=+0.399685785 container init bc9d1a1a82566d63bbe4bdf094bdee7f5f3555163a787f8b9309791ad075b4a5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-3-deactivate, release=754, vcs-type=git, name=centos-stream, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, maintainer=Guillaume Abrioux , ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., architecture=x86_64, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-04-07T14:49:45.597 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 07 14:49:45 smithi083 podman[67123]: 2024-04-07 14:49:45.422971898 +0000 UTC m=+0.402821131 container start bc9d1a1a82566d63bbe4bdf094bdee7f5f3555163a787f8b9309791ad075b4a5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-3-deactivate, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, vendor=Red Hat, Inc., io.buildah.version=1.19.8, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , vcs-type=git, distribution-scope=public, build-date=2022-05-03T08:36:31.336870) 2024-04-07T14:49:45.597 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 07 14:49:45 smithi083 podman[67123]: 2024-04-07 14:49:45.502969423 +0000 UTC m=+0.482818655 container attach bc9d1a1a82566d63bbe4bdf094bdee7f5f3555163a787f8b9309791ad075b4a5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-3-deactivate, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, version=8, RELEASE=HEAD, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , distribution-scope=public, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, release=754, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True) 2024-04-07T14:49:45.925 INFO:journalctl@ceph.osd.3.smithi083.stdout:Apr 07 14:49:45 smithi083 podman[67123]: 2024-04-07 14:49:45.718802222 +0000 UTC m=+0.698651466 container died bc9d1a1a82566d63bbe4bdf094bdee7f5f3555163a787f8b9309791ad075b4a5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-3-deactivate, io.openshift.expose-services=, RELEASE=HEAD, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, distribution-scope=public, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, name=centos-stream, GIT_CLEAN=True, version=8, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, ceph=True, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-07T14:49:46.090 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:49:45 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:49:45.837+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:12.776374+0000 front 2024-04-07T14:49:12.776258+0000 (oldest deadline 2024-04-07T14:49:44.576770+0000) 2024-04-07T14:49:46.090 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:49:45 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:49:45.893+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:16.385849+0000 front 2024-04-07T14:49:16.385969+0000 (oldest deadline 2024-04-07T14:49:42.286011+0000) 2024-04-07T14:49:46.352 DEBUG:teuthology.orchestra.run.smithi083:> sudo pkill -f 'journalctl -f -n 0 -u ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@osd.3.service' 2024-04-07T14:49:46.393 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-07T14:49:46.393 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-04-07T14:49:46.394 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-04-07T14:49:46.394 DEBUG:teuthology.orchestra.run.smithi189:> sudo systemctl stop ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@osd.4 2024-04-07T14:49:46.426 INFO:journalctl@ceph.osd.4.smithi189.stdout:Apr 07 14:49:46 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-4[30531]: 2024-04-07T14:49:46.186+0000 7f9536a9b700 -1 osd.4 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:18.122923+0000 front 2024-04-07T14:49:18.122987+0000 (oldest deadline 2024-04-07T14:49:41.622883+0000) 2024-04-07T14:49:46.427 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:49:46 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5[33437]: 2024-04-07T14:49:46.092+0000 7fd1945f7700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:18.211303+0000 front 2024-04-07T14:49:18.211329+0000 (oldest deadline 2024-04-07T14:49:43.511098+0000) 2024-04-07T14:49:46.708 INFO:journalctl@ceph.osd.4.smithi189.stdout:Apr 07 14:49:46 smithi189 systemd[1]: Stopping Ceph osd.4 for 7388281a-f4eb-11ee-b649-cb9ed24678a4... 2024-04-07T14:49:46.708 INFO:journalctl@ceph.osd.4.smithi189.stdout:Apr 07 14:49:46 smithi189 bash[56152]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd.4" found: no such container 2024-04-07T14:49:46.708 INFO:journalctl@ceph.osd.4.smithi189.stdout:Apr 07 14:49:46 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-4[30531]: 2024-04-07T14:49:46.572+0000 7f95390fe700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-07T14:49:46.708 INFO:journalctl@ceph.osd.4.smithi189.stdout:Apr 07 14:49:46 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-4[30531]: 2024-04-07T14:49:46.572+0000 7f95390fe700 -1 osd.4 97 *** Got signal Terminated *** 2024-04-07T14:49:46.708 INFO:journalctl@ceph.osd.4.smithi189.stdout:Apr 07 14:49:46 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-4[30531]: 2024-04-07T14:49:46.572+0000 7f95390fe700 -1 osd.4 97 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-07T14:49:47.093 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:49:46 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:49:46.885+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:16.385849+0000 front 2024-04-07T14:49:16.385969+0000 (oldest deadline 2024-04-07T14:49:42.286011+0000) 2024-04-07T14:49:47.094 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:49:46 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:49:46.824+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:12.776374+0000 front 2024-04-07T14:49:12.776258+0000 (oldest deadline 2024-04-07T14:49:44.576770+0000) 2024-04-07T14:49:47.457 INFO:journalctl@ceph.osd.4.smithi189.stdout:Apr 07 14:49:47 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-4[30531]: 2024-04-07T14:49:47.153+0000 7f9536a9b700 -1 osd.4 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:18.122923+0000 front 2024-04-07T14:49:18.122987+0000 (oldest deadline 2024-04-07T14:49:41.622883+0000) 2024-04-07T14:49:47.458 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:49:47 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5[33437]: 2024-04-07T14:49:47.095+0000 7fd1945f7700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:18.211303+0000 front 2024-04-07T14:49:18.211329+0000 (oldest deadline 2024-04-07T14:49:43.511098+0000) 2024-04-07T14:49:48.151 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:49:48 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5[33437]: 2024-04-07T14:49:48.060+0000 7fd1945f7700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:18.211303+0000 front 2024-04-07T14:49:18.211329+0000 (oldest deadline 2024-04-07T14:49:43.511098+0000) 2024-04-07T14:49:48.151 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:49:47 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:49:47.840+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:12.776374+0000 front 2024-04-07T14:49:12.776258+0000 (oldest deadline 2024-04-07T14:49:44.576770+0000) 2024-04-07T14:49:48.152 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:49:47 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:49:47.886+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:16.385849+0000 front 2024-04-07T14:49:16.385969+0000 (oldest deadline 2024-04-07T14:49:42.286011+0000) 2024-04-07T14:49:48.457 INFO:journalctl@ceph.osd.4.smithi189.stdout:Apr 07 14:49:48 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-4[30531]: 2024-04-07T14:49:48.153+0000 7f9536a9b700 -1 osd.4 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:18.122923+0000 front 2024-04-07T14:49:18.122987+0000 (oldest deadline 2024-04-07T14:49:41.622883+0000) 2024-04-07T14:49:49.182 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:49:49 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5[33437]: 2024-04-07T14:49:49.050+0000 7fd1945f7700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:18.211303+0000 front 2024-04-07T14:49:18.211329+0000 (oldest deadline 2024-04-07T14:49:43.511098+0000) 2024-04-07T14:49:49.183 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:49:48 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:49:48.935+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:16.385849+0000 front 2024-04-07T14:49:16.385969+0000 (oldest deadline 2024-04-07T14:49:42.286011+0000) 2024-04-07T14:49:49.183 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:49:48 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:49:48.830+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:12.776374+0000 front 2024-04-07T14:49:12.776258+0000 (oldest deadline 2024-04-07T14:49:44.576770+0000) 2024-04-07T14:49:49.457 INFO:journalctl@ceph.osd.4.smithi189.stdout:Apr 07 14:49:49 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-4[30531]: 2024-04-07T14:49:49.184+0000 7f9536a9b700 -1 osd.4 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:18.122923+0000 front 2024-04-07T14:49:18.122987+0000 (oldest deadline 2024-04-07T14:49:41.622883+0000) 2024-04-07T14:49:49.949 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:49:49 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:49:49.792+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:12.776374+0000 front 2024-04-07T14:49:12.776258+0000 (oldest deadline 2024-04-07T14:49:44.576770+0000) 2024-04-07T14:49:50.204 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:49:50 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5[33437]: 2024-04-07T14:49:50.053+0000 7fd1945f7700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:18.211303+0000 front 2024-04-07T14:49:18.211329+0000 (oldest deadline 2024-04-07T14:49:43.511098+0000) 2024-04-07T14:49:50.204 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:49:49 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:49:49.951+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:16.385849+0000 front 2024-04-07T14:49:16.385969+0000 (oldest deadline 2024-04-07T14:49:42.286011+0000) 2024-04-07T14:49:50.205 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:49:49 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:49:49.951+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:25.786770+0000 front 2024-04-07T14:49:25.786809+0000 (oldest deadline 2024-04-07T14:49:49.286702+0000) 2024-04-07T14:49:50.457 INFO:journalctl@ceph.osd.4.smithi189.stdout:Apr 07 14:49:50 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-4[30531]: 2024-04-07T14:49:50.205+0000 7f9536a9b700 -1 osd.4 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:18.122923+0000 front 2024-04-07T14:49:18.122987+0000 (oldest deadline 2024-04-07T14:49:41.622883+0000) 2024-04-07T14:49:51.208 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:49:50 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:49:50.771+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:12.776374+0000 front 2024-04-07T14:49:12.776258+0000 (oldest deadline 2024-04-07T14:49:44.576770+0000) 2024-04-07T14:49:51.208 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:49:50 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:49:50.771+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:24.577152+0000 front 2024-04-07T14:49:24.577247+0000 (oldest deadline 2024-04-07T14:49:50.477195+0000) 2024-04-07T14:49:51.208 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:49:51 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5[33437]: 2024-04-07T14:49:51.018+0000 7fd1945f7700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:18.211303+0000 front 2024-04-07T14:49:18.211329+0000 (oldest deadline 2024-04-07T14:49:43.511098+0000) 2024-04-07T14:49:51.208 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:49:51 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5[33437]: 2024-04-07T14:49:51.018+0000 7fd1945f7700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:27.612185+0000 front 2024-04-07T14:49:27.612133+0000 (oldest deadline 2024-04-07T14:49:50.512014+0000) 2024-04-07T14:49:51.209 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:49:50 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:49:50.967+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:16.385849+0000 front 2024-04-07T14:49:16.385969+0000 (oldest deadline 2024-04-07T14:49:42.286011+0000) 2024-04-07T14:49:51.209 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:49:50 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:49:50.967+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:25.786770+0000 front 2024-04-07T14:49:25.786809+0000 (oldest deadline 2024-04-07T14:49:49.286702+0000) 2024-04-07T14:49:51.707 INFO:journalctl@ceph.osd.4.smithi189.stdout:Apr 07 14:49:51 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-4[30531]: 2024-04-07T14:49:51.222+0000 7f9536a9b700 -1 osd.4 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:18.122923+0000 front 2024-04-07T14:49:18.122987+0000 (oldest deadline 2024-04-07T14:49:41.622883+0000) 2024-04-07T14:49:51.708 INFO:journalctl@ceph.osd.4.smithi189.stdout:Apr 07 14:49:51 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-4[30531]: 2024-04-07T14:49:51.222+0000 7f9536a9b700 -1 osd.4 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:25.123755+0000 front 2024-04-07T14:49:25.123670+0000 (oldest deadline 2024-04-07T14:49:51.023684+0000) 2024-04-07T14:49:51.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:49:51 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:49:51.656Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": dial tcp 172.21.15.83:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:49:51.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:49:51 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:49:51.656Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": dial tcp 172.21.15.83:8443: connect: connection refused" 2024-04-07T14:49:51.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:49:51 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:49:51.656Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:49:52.062 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:49:52 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5[33437]: 2024-04-07T14:49:52.004+0000 7fd1945f7700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:18.211303+0000 front 2024-04-07T14:49:18.211329+0000 (oldest deadline 2024-04-07T14:49:43.511098+0000) 2024-04-07T14:49:52.063 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:49:52 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5[33437]: 2024-04-07T14:49:52.004+0000 7fd1945f7700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:27.612185+0000 front 2024-04-07T14:49:27.612133+0000 (oldest deadline 2024-04-07T14:49:50.512014+0000) 2024-04-07T14:49:52.063 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:49:51 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:49:51.998+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:16.385849+0000 front 2024-04-07T14:49:16.385969+0000 (oldest deadline 2024-04-07T14:49:42.286011+0000) 2024-04-07T14:49:52.063 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:49:51 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:49:51.998+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:25.786770+0000 front 2024-04-07T14:49:25.786809+0000 (oldest deadline 2024-04-07T14:49:49.286702+0000) 2024-04-07T14:49:52.064 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:49:51 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:49:51.804+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:12.776374+0000 front 2024-04-07T14:49:12.776258+0000 (oldest deadline 2024-04-07T14:49:44.576770+0000) 2024-04-07T14:49:52.064 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:49:51 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:49:51.804+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:24.577152+0000 front 2024-04-07T14:49:24.577247+0000 (oldest deadline 2024-04-07T14:49:50.477195+0000) 2024-04-07T14:49:52.346 INFO:journalctl@ceph.osd.4.smithi189.stdout:Apr 07 14:49:52 smithi189 podman[56165]: 2024-04-07 14:49:52.064776518 +0000 UTC m=+5.580588553 container died 27e179b6db247c69a5f2186e695e70095034f95183e85a0a9e7dbbf392068053 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-4, build-date=2022-05-03T08:36:31.336870, release=754, GIT_BRANCH=HEAD, distribution-scope=public, maintainer=Guillaume Abrioux , version=8, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-07T14:49:52.707 INFO:journalctl@ceph.osd.4.smithi189.stdout:Apr 07 14:49:52 smithi189 podman[56165]: 2024-04-07 14:49:52.523029253 +0000 UTC m=+6.038841295 container cleanup 27e179b6db247c69a5f2186e695e70095034f95183e85a0a9e7dbbf392068053 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-4, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, vendor=Red Hat, Inc., architecture=x86_64, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, RELEASE=HEAD, vcs-type=git, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream) 2024-04-07T14:49:52.708 INFO:journalctl@ceph.osd.4.smithi189.stdout:Apr 07 14:49:52 smithi189 bash[56165]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-4 2024-04-07T14:49:52.708 INFO:journalctl@ceph.osd.4.smithi189.stdout:Apr 07 14:49:52 smithi189 bash[56197]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd.4" found: no such container 2024-04-07T14:49:53.037 INFO:journalctl@ceph.osd.4.smithi189.stdout:Apr 07 14:49:52 smithi189 podman[56183]: 2024-04-07 14:49:52.816117475 +0000 UTC m=+0.750076114 container remove 27e179b6db247c69a5f2186e695e70095034f95183e85a0a9e7dbbf392068053 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-4, io.k8s.display-name=CentOS Stream 8, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, io.buildah.version=1.19.8, architecture=x86_64, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, vcs-type=git, build-date=2022-05-03T08:36:31.336870, ceph=True, com.redhat.component=centos-stream-container, name=centos-stream, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, RELEASE=HEAD) 2024-04-07T14:49:53.037 INFO:journalctl@ceph.osd.4.smithi189.stdout:Apr 07 14:49:52 smithi189 podman[56262]: 2024-04-07 14:49:52.870888845 +0000 UTC m=+0.019187171 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-07T14:49:53.038 INFO:journalctl@ceph.osd.4.smithi189.stdout:Apr 07 14:49:52 smithi189 podman[56262]: 2024-04-07 14:49:52.983983042 +0000 UTC m=+0.132281387 container create 83f48112e3565c0cb7ae71cfd0749cf83e5c26d085244d0202f13e2f535df6ca (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-4-deactivate, release=754, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, vcs-type=git, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, name=centos-stream, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, ceph=True, io.openshift.expose-services=, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-07T14:49:53.038 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:49:53 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5[33437]: 2024-04-07T14:49:53.014+0000 7fd1945f7700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:18.211303+0000 front 2024-04-07T14:49:18.211329+0000 (oldest deadline 2024-04-07T14:49:43.511098+0000) 2024-04-07T14:49:53.038 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:49:53 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5[33437]: 2024-04-07T14:49:53.014+0000 7fd1945f7700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:27.612185+0000 front 2024-04-07T14:49:27.612133+0000 (oldest deadline 2024-04-07T14:49:50.512014+0000) 2024-04-07T14:49:53.038 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:49:52 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:49:52.766+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:12.776374+0000 front 2024-04-07T14:49:12.776258+0000 (oldest deadline 2024-04-07T14:49:44.576770+0000) 2024-04-07T14:49:53.039 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:49:52 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:49:52.766+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:24.577152+0000 front 2024-04-07T14:49:24.577247+0000 (oldest deadline 2024-04-07T14:49:50.477195+0000) 2024-04-07T14:49:53.292 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:49:53 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:49:53.037+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:16.385849+0000 front 2024-04-07T14:49:16.385969+0000 (oldest deadline 2024-04-07T14:49:42.286011+0000) 2024-04-07T14:49:53.292 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:49:53 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:49:53.037+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:25.786770+0000 front 2024-04-07T14:49:25.786809+0000 (oldest deadline 2024-04-07T14:49:49.286702+0000) 2024-04-07T14:49:53.293 INFO:journalctl@ceph.osd.4.smithi189.stdout:Apr 07 14:49:53 smithi189 podman[56262]: 2024-04-07 14:49:53.220007103 +0000 UTC m=+0.368305402 container init 83f48112e3565c0cb7ae71cfd0749cf83e5c26d085244d0202f13e2f535df6ca (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-4-deactivate, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, ceph=True, io.openshift.expose-services=, version=8, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754) 2024-04-07T14:49:53.293 INFO:journalctl@ceph.osd.4.smithi189.stdout:Apr 07 14:49:53 smithi189 podman[56262]: 2024-04-07 14:49:53.223251731 +0000 UTC m=+0.371550030 container start 83f48112e3565c0cb7ae71cfd0749cf83e5c26d085244d0202f13e2f535df6ca (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-4-deactivate, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, ceph=True, io.openshift.expose-services=, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, vcs-type=git) 2024-04-07T14:49:53.293 INFO:journalctl@ceph.osd.4.smithi189.stdout:Apr 07 14:49:53 smithi189 podman[56262]: 2024-04-07 14:49:53.294905653 +0000 UTC m=+0.443203961 container attach 83f48112e3565c0cb7ae71cfd0749cf83e5c26d085244d0202f13e2f535df6ca (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-4-deactivate, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, vcs-type=git, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , release=754, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, RELEASE=HEAD, architecture=x86_64, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, GIT_CLEAN=True) 2024-04-07T14:49:53.707 INFO:journalctl@ceph.osd.4.smithi189.stdout:Apr 07 14:49:53 smithi189 podman[56262]: 2024-04-07 14:49:53.513015448 +0000 UTC m=+0.661313750 container died 83f48112e3565c0cb7ae71cfd0749cf83e5c26d085244d0202f13e2f535df6ca (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-4-deactivate, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, com.redhat.component=centos-stream-container, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, release=754, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, ceph=True, io.openshift.expose-services=, vcs-type=git, io.buildah.version=1.19.8, distribution-scope=public, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, architecture=x86_64) 2024-04-07T14:49:53.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:49:53 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:49:53.537Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": dial tcp 172.21.15.83:8443: connect: connection refused" 2024-04-07T14:49:53.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:49:53 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:49:53.537Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": dial tcp 172.21.15.83:8443: connect: connection refused" 2024-04-07T14:49:53.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:49:53 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:49:53.537Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:49:54.072 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:49:53 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5[33437]: 2024-04-07T14:49:53.966+0000 7fd1945f7700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:18.211303+0000 front 2024-04-07T14:49:18.211329+0000 (oldest deadline 2024-04-07T14:49:43.511098+0000) 2024-04-07T14:49:54.073 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:49:53 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5[33437]: 2024-04-07T14:49:53.967+0000 7fd1945f7700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:27.612185+0000 front 2024-04-07T14:49:27.612133+0000 (oldest deadline 2024-04-07T14:49:50.512014+0000) 2024-04-07T14:49:54.073 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:49:53 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:49:53.748+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:12.776374+0000 front 2024-04-07T14:49:12.776258+0000 (oldest deadline 2024-04-07T14:49:44.576770+0000) 2024-04-07T14:49:54.073 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:49:53 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:49:53.748+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:24.577152+0000 front 2024-04-07T14:49:24.577247+0000 (oldest deadline 2024-04-07T14:49:50.477195+0000) 2024-04-07T14:49:54.124 DEBUG:teuthology.orchestra.run.smithi189:> sudo pkill -f 'journalctl -f -n 0 -u ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@osd.4.service' 2024-04-07T14:49:54.208 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-07T14:49:54.208 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-04-07T14:49:54.208 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-04-07T14:49:54.208 DEBUG:teuthology.orchestra.run.smithi189:> sudo systemctl stop ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@osd.5 2024-04-07T14:49:54.370 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:49:54 smithi189 systemd[1]: Stopping Ceph osd.5 for 7388281a-f4eb-11ee-b649-cb9ed24678a4... 2024-04-07T14:49:54.370 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:49:54 smithi189 bash[56341]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd.5" found: no such container 2024-04-07T14:49:54.371 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:49:54 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:49:54.074+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:16.385849+0000 front 2024-04-07T14:49:16.385969+0000 (oldest deadline 2024-04-07T14:49:42.286011+0000) 2024-04-07T14:49:54.371 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:49:54 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:49:54.074+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:25.786770+0000 front 2024-04-07T14:49:25.786809+0000 (oldest deadline 2024-04-07T14:49:49.286702+0000) 2024-04-07T14:49:54.707 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:49:54 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5[33437]: 2024-04-07T14:49:54.451+0000 7fd19945f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-07T14:49:54.707 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:49:54 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5[33437]: 2024-04-07T14:49:54.451+0000 7fd19945f700 -1 osd.5 97 *** Got signal Terminated *** 2024-04-07T14:49:54.708 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:49:54 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5[33437]: 2024-04-07T14:49:54.451+0000 7fd19945f700 -1 osd.5 97 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-07T14:49:55.056 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:49:54 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5[33437]: 2024-04-07T14:49:54.960+0000 7fd1945f7700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:18.211303+0000 front 2024-04-07T14:49:18.211329+0000 (oldest deadline 2024-04-07T14:49:43.511098+0000) 2024-04-07T14:49:55.057 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:49:54 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5[33437]: 2024-04-07T14:49:54.960+0000 7fd1945f7700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:27.612185+0000 front 2024-04-07T14:49:27.612133+0000 (oldest deadline 2024-04-07T14:49:50.512014+0000) 2024-04-07T14:49:55.057 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:49:54 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:49:54.763+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:12.776374+0000 front 2024-04-07T14:49:12.776258+0000 (oldest deadline 2024-04-07T14:49:44.576770+0000) 2024-04-07T14:49:55.057 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:49:54 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:49:54.763+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:24.577152+0000 front 2024-04-07T14:49:24.577247+0000 (oldest deadline 2024-04-07T14:49:50.477195+0000) 2024-04-07T14:49:55.458 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:49:55 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:49:55.058+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:16.385849+0000 front 2024-04-07T14:49:16.385969+0000 (oldest deadline 2024-04-07T14:49:42.286011+0000) 2024-04-07T14:49:55.458 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:49:55 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:49:55.058+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:25.786770+0000 front 2024-04-07T14:49:25.786809+0000 (oldest deadline 2024-04-07T14:49:49.286702+0000) 2024-04-07T14:49:56.068 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:49:55 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5[33437]: 2024-04-07T14:49:55.933+0000 7fd1945f7700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:18.211303+0000 front 2024-04-07T14:49:18.211329+0000 (oldest deadline 2024-04-07T14:49:43.511098+0000) 2024-04-07T14:49:56.068 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:49:55 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5[33437]: 2024-04-07T14:49:55.933+0000 7fd1945f7700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:27.612185+0000 front 2024-04-07T14:49:27.612133+0000 (oldest deadline 2024-04-07T14:49:50.512014+0000) 2024-04-07T14:49:56.068 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:49:55 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:49:55.783+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:12.776374+0000 front 2024-04-07T14:49:12.776258+0000 (oldest deadline 2024-04-07T14:49:44.576770+0000) 2024-04-07T14:49:56.068 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:49:55 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:49:55.783+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:24.577152+0000 front 2024-04-07T14:49:24.577247+0000 (oldest deadline 2024-04-07T14:49:50.477195+0000) 2024-04-07T14:49:56.458 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:49:56 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:49:56.070+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:16.385849+0000 front 2024-04-07T14:49:16.385969+0000 (oldest deadline 2024-04-07T14:49:42.286011+0000) 2024-04-07T14:49:56.458 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:49:56 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:49:56.070+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:25.786770+0000 front 2024-04-07T14:49:25.786809+0000 (oldest deadline 2024-04-07T14:49:49.286702+0000) 2024-04-07T14:49:57.009 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:49:56 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5[33437]: 2024-04-07T14:49:56.937+0000 7fd1945f7700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:18.211303+0000 front 2024-04-07T14:49:18.211329+0000 (oldest deadline 2024-04-07T14:49:43.511098+0000) 2024-04-07T14:49:57.009 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:49:56 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5[33437]: 2024-04-07T14:49:56.937+0000 7fd1945f7700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:27.612185+0000 front 2024-04-07T14:49:27.612133+0000 (oldest deadline 2024-04-07T14:49:50.512014+0000) 2024-04-07T14:49:57.009 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:49:56 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:49:56.735+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:12.776374+0000 front 2024-04-07T14:49:12.776258+0000 (oldest deadline 2024-04-07T14:49:44.576770+0000) 2024-04-07T14:49:57.010 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:49:56 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:49:56.735+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:24.577152+0000 front 2024-04-07T14:49:24.577247+0000 (oldest deadline 2024-04-07T14:49:50.477195+0000) 2024-04-07T14:49:57.303 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:49:57 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:49:57.051+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:16.385849+0000 front 2024-04-07T14:49:16.385969+0000 (oldest deadline 2024-04-07T14:49:42.286011+0000) 2024-04-07T14:49:57.304 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:49:57 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:49:57.051+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:25.786770+0000 front 2024-04-07T14:49:25.786809+0000 (oldest deadline 2024-04-07T14:49:49.286702+0000) 2024-04-07T14:49:57.958 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:49:57 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:49:57.718+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:12.776374+0000 front 2024-04-07T14:49:12.776258+0000 (oldest deadline 2024-04-07T14:49:44.576770+0000) 2024-04-07T14:49:57.958 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:49:57 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:49:57.718+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:24.577152+0000 front 2024-04-07T14:49:24.577247+0000 (oldest deadline 2024-04-07T14:49:50.477195+0000) 2024-04-07T14:49:58.458 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:49:57 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5[33437]: 2024-04-07T14:49:57.961+0000 7fd1945f7700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:18.211303+0000 front 2024-04-07T14:49:18.211329+0000 (oldest deadline 2024-04-07T14:49:43.511098+0000) 2024-04-07T14:49:58.458 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:49:57 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5[33437]: 2024-04-07T14:49:57.961+0000 7fd1945f7700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:27.612185+0000 front 2024-04-07T14:49:27.612133+0000 (oldest deadline 2024-04-07T14:49:50.512014+0000) 2024-04-07T14:49:58.459 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:49:58 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:49:58.062+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:16.385849+0000 front 2024-04-07T14:49:16.385969+0000 (oldest deadline 2024-04-07T14:49:42.286011+0000) 2024-04-07T14:49:58.459 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:49:58 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:49:58.062+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:25.786770+0000 front 2024-04-07T14:49:25.786809+0000 (oldest deadline 2024-04-07T14:49:49.286702+0000) 2024-04-07T14:49:58.459 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:49:58 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:49:58.062+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-07T14:49:35.088316+0000 front 2024-04-07T14:49:35.088344+0000 (oldest deadline 2024-04-07T14:49:57.388276+0000) 2024-04-07T14:49:58.998 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:49:58 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:49:58.741+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:12.776374+0000 front 2024-04-07T14:49:12.776258+0000 (oldest deadline 2024-04-07T14:49:44.576770+0000) 2024-04-07T14:49:58.998 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:49:58 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:49:58.741+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:24.577152+0000 front 2024-04-07T14:49:24.577247+0000 (oldest deadline 2024-04-07T14:49:50.477195+0000) 2024-04-07T14:49:58.998 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:49:58 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:49:58.741+0000 7f7c963ef700 -1 osd.7 97 get_health_metrics reporting 1 slow ops, oldest is osd_op(client.24637.0:128 2.0 2:02bc1a72:::realms.3a89d6cc-37c5-47ad-85a8-f8c0285f9dde.control:head [watch ping cookie 94849229681152] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e97) 2024-04-07T14:49:59.458 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:49:59 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5[33437]: 2024-04-07T14:49:59.000+0000 7fd1945f7700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:18.211303+0000 front 2024-04-07T14:49:18.211329+0000 (oldest deadline 2024-04-07T14:49:43.511098+0000) 2024-04-07T14:49:59.458 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:49:59 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5[33437]: 2024-04-07T14:49:59.000+0000 7fd1945f7700 -1 osd.5 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:27.612185+0000 front 2024-04-07T14:49:27.612133+0000 (oldest deadline 2024-04-07T14:49:50.512014+0000) 2024-04-07T14:49:59.458 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:49:59 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:49:59.017+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:16.385849+0000 front 2024-04-07T14:49:16.385969+0000 (oldest deadline 2024-04-07T14:49:42.286011+0000) 2024-04-07T14:49:59.458 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:49:59 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:49:59.017+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:25.786770+0000 front 2024-04-07T14:49:25.786809+0000 (oldest deadline 2024-04-07T14:49:49.286702+0000) 2024-04-07T14:49:59.459 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:49:59 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:49:59.017+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-07T14:49:35.088316+0000 front 2024-04-07T14:49:35.088344+0000 (oldest deadline 2024-04-07T14:49:57.388276+0000) 2024-04-07T14:49:59.958 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:49:59 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:49:59.707+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:12.776374+0000 front 2024-04-07T14:49:12.776258+0000 (oldest deadline 2024-04-07T14:49:44.576770+0000) 2024-04-07T14:49:59.958 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:49:59 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:49:59.707+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:24.577152+0000 front 2024-04-07T14:49:24.577247+0000 (oldest deadline 2024-04-07T14:49:50.477195+0000) 2024-04-07T14:49:59.958 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:49:59 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:49:59.707+0000 7f7c963ef700 -1 osd.7 97 get_health_metrics reporting 1 slow ops, oldest is osd_op(client.24637.0:128 2.0 2:02bc1a72:::realms.3a89d6cc-37c5-47ad-85a8-f8c0285f9dde.control:head [watch ping cookie 94849229681152] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e97) 2024-04-07T14:49:59.958 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:49:59 smithi189 podman[56352]: 2024-04-07 14:49:59.809123392 +0000 UTC m=+5.487120997 container stop 9957f4f08073eabedb3562912c3a1d4ccb5ad635a519b73e1234f3efdaf29d99 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5, ceph=True, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, vcs-type=git, maintainer=Guillaume Abrioux , version=8, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=HEAD, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc.) 2024-04-07T14:49:59.958 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:49:59 smithi189 podman[56352]: 2024-04-07 14:49:59.834987157 +0000 UTC m=+5.512984751 container died 9957f4f08073eabedb3562912c3a1d4ccb5ad635a519b73e1234f3efdaf29d99 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, vendor=Red Hat, Inc., vcs-type=git, io.buildah.version=1.19.8, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, RELEASE=HEAD, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8) 2024-04-07T14:50:00.441 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:50:00 smithi189 podman[56352]: 2024-04-07 14:50:00.191848339 +0000 UTC m=+5.869845932 container cleanup 9957f4f08073eabedb3562912c3a1d4ccb5ad635a519b73e1234f3efdaf29d99 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, ceph=True, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., distribution-scope=public, architecture=x86_64, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, version=8) 2024-04-07T14:50:00.441 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:50:00 smithi189 bash[56352]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5 2024-04-07T14:50:00.441 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:50:00 smithi189 bash[56437]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd.5" found: no such container 2024-04-07T14:50:00.442 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:50:00 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:50:00.053+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:16.385849+0000 front 2024-04-07T14:49:16.385969+0000 (oldest deadline 2024-04-07T14:49:42.286011+0000) 2024-04-07T14:50:00.442 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:50:00 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:50:00.053+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:25.786770+0000 front 2024-04-07T14:49:25.786809+0000 (oldest deadline 2024-04-07T14:49:49.286702+0000) 2024-04-07T14:50:00.442 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:50:00 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:50:00.053+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-07T14:49:35.088316+0000 front 2024-04-07T14:49:35.088344+0000 (oldest deadline 2024-04-07T14:49:57.388276+0000) 2024-04-07T14:50:00.697 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:50:00 smithi189 podman[56420]: 2024-04-07 14:50:00.443466383 +0000 UTC m=+0.626672247 container remove 9957f4f08073eabedb3562912c3a1d4ccb5ad635a519b73e1234f3efdaf29d99 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , release=754, name=centos-stream, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, distribution-scope=public, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, vcs-type=git, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc.) 2024-04-07T14:50:00.698 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:50:00 smithi189 podman[56501]: 2024-04-07 14:50:00.534932469 +0000 UTC m=+0.018883355 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-07T14:50:00.698 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:50:00 smithi189 podman[56501]: 2024-04-07 14:50:00.652705613 +0000 UTC m=+0.136656494 container create 4dffeb4dc372daac02cfdd9dccbc96b19c9ef90c62a18a1ef349916ab61dffce (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5-deactivate, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, io.buildah.version=1.19.8, io.openshift.expose-services=, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, version=8, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., architecture=x86_64) 2024-04-07T14:50:00.958 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:00 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:00.700+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:12.776374+0000 front 2024-04-07T14:49:12.776258+0000 (oldest deadline 2024-04-07T14:49:44.576770+0000) 2024-04-07T14:50:00.958 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:00 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:00.700+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:24.577152+0000 front 2024-04-07T14:49:24.577247+0000 (oldest deadline 2024-04-07T14:49:50.477195+0000) 2024-04-07T14:50:00.958 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:00 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:00.700+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-07T14:49:35.678711+0000 front 2024-04-07T14:49:35.678735+0000 (oldest deadline 2024-04-07T14:50:00.378605+0000) 2024-04-07T14:50:00.958 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:00 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:00.700+0000 7f7c963ef700 -1 osd.7 97 get_health_metrics reporting 1 slow ops, oldest is osd_op(client.24637.0:128 2.0 2:02bc1a72:::realms.3a89d6cc-37c5-47ad-85a8-f8c0285f9dde.control:head [watch ping cookie 94849229681152] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e97) 2024-04-07T14:50:00.958 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:50:00 smithi189 podman[56501]: 2024-04-07 14:50:00.880907866 +0000 UTC m=+0.364858858 container init 4dffeb4dc372daac02cfdd9dccbc96b19c9ef90c62a18a1ef349916ab61dffce (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5-deactivate, architecture=x86_64, com.redhat.component=centos-stream-container, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, version=8, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, io.openshift.expose-services=) 2024-04-07T14:50:00.959 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:50:00 smithi189 podman[56501]: 2024-04-07 14:50:00.883981687 +0000 UTC m=+0.367932571 container start 4dffeb4dc372daac02cfdd9dccbc96b19c9ef90c62a18a1ef349916ab61dffce (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5-deactivate, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, version=8, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, GIT_CLEAN=True, architecture=x86_64, GIT_BRANCH=HEAD, RELEASE=HEAD, ceph=True, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-07T14:50:01.458 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:50:01 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:50:01.092+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:16.385849+0000 front 2024-04-07T14:49:16.385969+0000 (oldest deadline 2024-04-07T14:49:42.286011+0000) 2024-04-07T14:50:01.458 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:50:01 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:50:01.092+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:25.786770+0000 front 2024-04-07T14:49:25.786809+0000 (oldest deadline 2024-04-07T14:49:49.286702+0000) 2024-04-07T14:50:01.458 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:50:01 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:50:01.092+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-07T14:49:35.088316+0000 front 2024-04-07T14:49:35.088344+0000 (oldest deadline 2024-04-07T14:49:57.388276+0000) 2024-04-07T14:50:01.458 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:50:00 smithi189 podman[56501]: 2024-04-07 14:50:00.963415871 +0000 UTC m=+0.447366761 container attach 4dffeb4dc372daac02cfdd9dccbc96b19c9ef90c62a18a1ef349916ab61dffce (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5-deactivate, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, vcs-type=git, ceph=True, name=centos-stream, io.openshift.expose-services=, RELEASE=HEAD, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream) 2024-04-07T14:50:01.459 INFO:journalctl@ceph.osd.5.smithi189.stdout:Apr 07 14:50:01 smithi189 podman[56501]: 2024-04-07 14:50:01.170295647 +0000 UTC m=+0.654246527 container died 4dffeb4dc372daac02cfdd9dccbc96b19c9ef90c62a18a1ef349916ab61dffce (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-5-deactivate, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, release=754, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, version=8, RELEASE=HEAD, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-07T14:50:01.731 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:01 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:01.682+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:12.776374+0000 front 2024-04-07T14:49:12.776258+0000 (oldest deadline 2024-04-07T14:49:44.576770+0000) 2024-04-07T14:50:01.731 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:01 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:01.682+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:24.577152+0000 front 2024-04-07T14:49:24.577247+0000 (oldest deadline 2024-04-07T14:49:50.477195+0000) 2024-04-07T14:50:01.731 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:01 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:01.682+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-07T14:49:35.678711+0000 front 2024-04-07T14:49:35.678735+0000 (oldest deadline 2024-04-07T14:50:00.378605+0000) 2024-04-07T14:50:01.731 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:01 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:01.682+0000 7f7c963ef700 -1 osd.7 97 get_health_metrics reporting 2 slow ops, oldest is osd_op(client.24637.0:128 2.0 2:02bc1a72:::realms.3a89d6cc-37c5-47ad-85a8-f8c0285f9dde.control:head [watch ping cookie 94849229681152] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e97) 2024-04-07T14:50:01.754 DEBUG:teuthology.orchestra.run.smithi189:> sudo pkill -f 'journalctl -f -n 0 -u ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@osd.5.service' 2024-04-07T14:50:01.795 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-07T14:50:01.796 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-04-07T14:50:01.796 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-04-07T14:50:01.796 DEBUG:teuthology.orchestra.run.smithi189:> sudo systemctl stop ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@osd.6 2024-04-07T14:50:01.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:50:01 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:50:01.657Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": dial tcp 172.21.15.83:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:50:01.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:50:01 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:50:01.657Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": dial tcp 172.21.15.83:8443: connect: connection refused" 2024-04-07T14:50:01.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:50:01 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:50:01.657Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:50:02.026 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:50:01 smithi189 systemd[1]: Stopping Ceph osd.6 for 7388281a-f4eb-11ee-b649-cb9ed24678a4... 2024-04-07T14:50:02.027 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:50:01 smithi189 bash[56581]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd.6" found: no such container 2024-04-07T14:50:02.027 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:50:02 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:50:02.027+0000 7efca97f5700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-07T14:50:02.027 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:50:02 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:50:02.027+0000 7efca97f5700 -1 osd.6 97 *** Got signal Terminated *** 2024-04-07T14:50:02.458 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:50:02 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:50:02.027+0000 7efca97f5700 -1 osd.6 97 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-07T14:50:02.458 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:50:02 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:50:02.113+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:16.385849+0000 front 2024-04-07T14:49:16.385969+0000 (oldest deadline 2024-04-07T14:49:42.286011+0000) 2024-04-07T14:50:02.458 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:50:02 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:50:02.113+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:25.786770+0000 front 2024-04-07T14:49:25.786809+0000 (oldest deadline 2024-04-07T14:49:49.286702+0000) 2024-04-07T14:50:02.458 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:50:02 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:50:02.113+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-07T14:49:35.088316+0000 front 2024-04-07T14:49:35.088344+0000 (oldest deadline 2024-04-07T14:49:57.388276+0000) 2024-04-07T14:50:02.957 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:02 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:02.690+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:12.776374+0000 front 2024-04-07T14:49:12.776258+0000 (oldest deadline 2024-04-07T14:49:44.576770+0000) 2024-04-07T14:50:02.958 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:02 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:02.690+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:24.577152+0000 front 2024-04-07T14:49:24.577247+0000 (oldest deadline 2024-04-07T14:49:50.477195+0000) 2024-04-07T14:50:02.958 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:02 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:02.690+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-07T14:49:35.678711+0000 front 2024-04-07T14:49:35.678735+0000 (oldest deadline 2024-04-07T14:50:00.378605+0000) 2024-04-07T14:50:02.958 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:02 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:02.690+0000 7f7c963ef700 -1 osd.7 97 get_health_metrics reporting 2 slow ops, oldest is osd_op(client.24637.0:128 2.0 2:02bc1a72:::realms.3a89d6cc-37c5-47ad-85a8-f8c0285f9dde.control:head [watch ping cookie 94849229681152] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e97) 2024-04-07T14:50:03.457 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:50:03 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:50:03.082+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:16.385849+0000 front 2024-04-07T14:49:16.385969+0000 (oldest deadline 2024-04-07T14:49:42.286011+0000) 2024-04-07T14:50:03.458 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:50:03 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:50:03.082+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:25.786770+0000 front 2024-04-07T14:49:25.786809+0000 (oldest deadline 2024-04-07T14:49:49.286702+0000) 2024-04-07T14:50:03.458 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:50:03 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:50:03.082+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-07T14:49:35.088316+0000 front 2024-04-07T14:49:35.088344+0000 (oldest deadline 2024-04-07T14:49:57.388276+0000) 2024-04-07T14:50:03.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:50:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:50:03.538Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": dial tcp 172.21.15.83:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:50:03.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:50:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:50:03.538Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": dial tcp 172.21.15.83:8443: connect: connection refused" 2024-04-07T14:50:03.926 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:50:03 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:50:03.538Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:50:03.958 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:03 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:03.684+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:12.776374+0000 front 2024-04-07T14:49:12.776258+0000 (oldest deadline 2024-04-07T14:49:44.576770+0000) 2024-04-07T14:50:03.958 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:03 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:03.684+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:24.577152+0000 front 2024-04-07T14:49:24.577247+0000 (oldest deadline 2024-04-07T14:49:50.477195+0000) 2024-04-07T14:50:03.958 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:03 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:03.684+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-07T14:49:35.678711+0000 front 2024-04-07T14:49:35.678735+0000 (oldest deadline 2024-04-07T14:50:00.378605+0000) 2024-04-07T14:50:03.958 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:03 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:03.684+0000 7f7c963ef700 -1 osd.7 97 get_health_metrics reporting 3 slow ops, oldest is osd_op(client.24637.0:128 2.0 2:02bc1a72:::realms.3a89d6cc-37c5-47ad-85a8-f8c0285f9dde.control:head [watch ping cookie 94849229681152] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e97) 2024-04-07T14:50:04.457 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:50:04 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:50:04.054+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:16.385849+0000 front 2024-04-07T14:49:16.385969+0000 (oldest deadline 2024-04-07T14:49:42.286011+0000) 2024-04-07T14:50:04.458 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:50:04 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:50:04.054+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:25.786770+0000 front 2024-04-07T14:49:25.786809+0000 (oldest deadline 2024-04-07T14:49:49.286702+0000) 2024-04-07T14:50:04.458 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:50:04 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:50:04.054+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-07T14:49:35.088316+0000 front 2024-04-07T14:49:35.088344+0000 (oldest deadline 2024-04-07T14:49:57.388276+0000) 2024-04-07T14:50:04.957 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:04 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:04.698+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:12.776374+0000 front 2024-04-07T14:49:12.776258+0000 (oldest deadline 2024-04-07T14:49:44.576770+0000) 2024-04-07T14:50:04.958 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:04 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:04.698+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:24.577152+0000 front 2024-04-07T14:49:24.577247+0000 (oldest deadline 2024-04-07T14:49:50.477195+0000) 2024-04-07T14:50:04.958 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:04 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:04.698+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-07T14:49:35.678711+0000 front 2024-04-07T14:49:35.678735+0000 (oldest deadline 2024-04-07T14:50:00.378605+0000) 2024-04-07T14:50:04.958 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:04 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:04.698+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-04-07T14:49:40.379331+0000 front 2024-04-07T14:49:40.379277+0000 (oldest deadline 2024-04-07T14:50:04.479122+0000) 2024-04-07T14:50:04.958 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:04 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:04.698+0000 7f7c963ef700 -1 osd.7 97 get_health_metrics reporting 3 slow ops, oldest is osd_op(client.24637.0:128 2.0 2:02bc1a72:::realms.3a89d6cc-37c5-47ad-85a8-f8c0285f9dde.control:head [watch ping cookie 94849229681152] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e97) 2024-04-07T14:50:05.354 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:50:05 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:50:05.100+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:16.385849+0000 front 2024-04-07T14:49:16.385969+0000 (oldest deadline 2024-04-07T14:49:42.286011+0000) 2024-04-07T14:50:05.354 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:50:05 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:50:05.100+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:25.786770+0000 front 2024-04-07T14:49:25.786809+0000 (oldest deadline 2024-04-07T14:49:49.286702+0000) 2024-04-07T14:50:05.354 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:50:05 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:50:05.100+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-07T14:49:35.088316+0000 front 2024-04-07T14:49:35.088344+0000 (oldest deadline 2024-04-07T14:49:57.388276+0000) 2024-04-07T14:50:05.957 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:05 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:05.683+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:12.776374+0000 front 2024-04-07T14:49:12.776258+0000 (oldest deadline 2024-04-07T14:49:44.576770+0000) 2024-04-07T14:50:05.958 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:05 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:05.683+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:24.577152+0000 front 2024-04-07T14:49:24.577247+0000 (oldest deadline 2024-04-07T14:49:50.477195+0000) 2024-04-07T14:50:05.958 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:05 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:05.683+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-07T14:49:35.678711+0000 front 2024-04-07T14:49:35.678735+0000 (oldest deadline 2024-04-07T14:50:00.378605+0000) 2024-04-07T14:50:05.958 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:05 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:05.683+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-04-07T14:49:40.379331+0000 front 2024-04-07T14:49:40.379277+0000 (oldest deadline 2024-04-07T14:50:04.479122+0000) 2024-04-07T14:50:05.958 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:05 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:05.683+0000 7f7c963ef700 -1 osd.7 97 get_health_metrics reporting 3 slow ops, oldest is osd_op(client.24637.0:128 2.0 2:02bc1a72:::realms.3a89d6cc-37c5-47ad-85a8-f8c0285f9dde.control:head [watch ping cookie 94849229681152] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e97) 2024-04-07T14:50:06.411 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:50:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:50:06.149+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:16.385849+0000 front 2024-04-07T14:49:16.385969+0000 (oldest deadline 2024-04-07T14:49:42.286011+0000) 2024-04-07T14:50:06.411 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:50:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:50:06.149+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:25.786770+0000 front 2024-04-07T14:49:25.786809+0000 (oldest deadline 2024-04-07T14:49:49.286702+0000) 2024-04-07T14:50:06.412 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:50:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:50:06.149+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-07T14:49:35.088316+0000 front 2024-04-07T14:49:35.088344+0000 (oldest deadline 2024-04-07T14:49:57.388276+0000) 2024-04-07T14:50:06.412 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:50:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6[36342]: 2024-04-07T14:50:06.149+0000 7efca598f700 -1 osd.6 97 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-04-07T14:49:42.689355+0000 front 2024-04-07T14:49:42.689329+0000 (oldest deadline 2024-04-07T14:50:05.589258+0000) 2024-04-07T14:50:07.123 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:06.724+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:12.776374+0000 front 2024-04-07T14:49:12.776258+0000 (oldest deadline 2024-04-07T14:49:44.576770+0000) 2024-04-07T14:50:07.124 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:06.724+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:24.577152+0000 front 2024-04-07T14:49:24.577247+0000 (oldest deadline 2024-04-07T14:49:50.477195+0000) 2024-04-07T14:50:07.124 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:06.724+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-07T14:49:35.678711+0000 front 2024-04-07T14:49:35.678735+0000 (oldest deadline 2024-04-07T14:50:00.378605+0000) 2024-04-07T14:50:07.124 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:06.724+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-04-07T14:49:40.379331+0000 front 2024-04-07T14:49:40.379277+0000 (oldest deadline 2024-04-07T14:50:04.479122+0000) 2024-04-07T14:50:07.124 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:06 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:06.724+0000 7f7c963ef700 -1 osd.7 97 get_health_metrics reporting 4 slow ops, oldest is osd_op(client.24637.0:128 2.0 2:02bc1a72:::realms.3a89d6cc-37c5-47ad-85a8-f8c0285f9dde.control:head [watch ping cookie 94849229681152] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e97) 2024-04-07T14:50:07.764 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:50:07 smithi189 podman[56593]: 2024-04-07 14:50:07.503065611 +0000 UTC m=+5.593097203 container died af997398de2d3a8b3f1b1928d93ba2841a255c3395449e572efdc19502cb61c1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, GIT_CLEAN=True, RELEASE=HEAD, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, ceph=True, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0) 2024-04-07T14:50:08.120 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:50:07 smithi189 podman[56593]: 2024-04-07 14:50:07.89680131 +0000 UTC m=+5.986832895 container cleanup af997398de2d3a8b3f1b1928d93ba2841a255c3395449e572efdc19502cb61c1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, CEPH_POINT_RELEASE=-17.2.0, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, vendor=Red Hat, Inc., name=centos-stream, io.openshift.expose-services=, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, RELEASE=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD) 2024-04-07T14:50:08.121 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:50:07 smithi189 bash[56593]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6 2024-04-07T14:50:08.121 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:50:07 smithi189 bash[57161]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd.6" found: no such container 2024-04-07T14:50:08.121 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:07 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:07.765+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:12.776374+0000 front 2024-04-07T14:49:12.776258+0000 (oldest deadline 2024-04-07T14:49:44.576770+0000) 2024-04-07T14:50:08.121 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:07 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:07.765+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:24.577152+0000 front 2024-04-07T14:49:24.577247+0000 (oldest deadline 2024-04-07T14:49:50.477195+0000) 2024-04-07T14:50:08.121 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:07 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:07.765+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-07T14:49:35.678711+0000 front 2024-04-07T14:49:35.678735+0000 (oldest deadline 2024-04-07T14:50:00.378605+0000) 2024-04-07T14:50:08.121 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:07 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:07.765+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-04-07T14:49:40.379331+0000 front 2024-04-07T14:49:40.379277+0000 (oldest deadline 2024-04-07T14:50:04.479122+0000) 2024-04-07T14:50:08.122 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:07 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:07.765+0000 7f7c963ef700 -1 osd.7 97 get_health_metrics reporting 4 slow ops, oldest is osd_op(client.24637.0:128 2.0 2:02bc1a72:::realms.3a89d6cc-37c5-47ad-85a8-f8c0285f9dde.control:head [watch ping cookie 94849229681152] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e97) 2024-04-07T14:50:08.447 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:50:08 smithi189 podman[57131]: 2024-04-07 14:50:08.123111884 +0000 UTC m=+0.624040591 container remove af997398de2d3a8b3f1b1928d93ba2841a255c3395449e572efdc19502cb61c1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, ceph=True, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, version=8, vcs-type=git, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, release=754, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-04-07T14:50:08.447 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:50:08 smithi189 podman[57239]: 2024-04-07 14:50:08.243100151 +0000 UTC m=+0.020843169 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-07T14:50:09.130 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:50:09 smithi189 podman[57239]: 2024-04-07 14:50:09.0111804 +0000 UTC m=+0.788923403 container create 0d73f39d792d88723a6b4d61f26ac9a2ca6ee1e19eb441f021b2aa90e97631fc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6-deactivate, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , release=754, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, build-date=2022-05-03T08:36:31.336870, ceph=True, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, GIT_CLEAN=True, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc.) 2024-04-07T14:50:09.131 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:08 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:08.811+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:12.776374+0000 front 2024-04-07T14:49:12.776258+0000 (oldest deadline 2024-04-07T14:49:44.576770+0000) 2024-04-07T14:50:09.131 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:08 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:08.811+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:24.577152+0000 front 2024-04-07T14:49:24.577247+0000 (oldest deadline 2024-04-07T14:49:50.477195+0000) 2024-04-07T14:50:09.131 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:08 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:08.811+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-07T14:49:35.678711+0000 front 2024-04-07T14:49:35.678735+0000 (oldest deadline 2024-04-07T14:50:00.378605+0000) 2024-04-07T14:50:09.131 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:08 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:08.811+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-04-07T14:49:40.379331+0000 front 2024-04-07T14:49:40.379277+0000 (oldest deadline 2024-04-07T14:50:04.479122+0000) 2024-04-07T14:50:09.131 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:08 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:08.812+0000 7f7c963ef700 -1 osd.7 97 get_health_metrics reporting 5 slow ops, oldest is osd_op(client.24637.0:128 2.0 2:02bc1a72:::realms.3a89d6cc-37c5-47ad-85a8-f8c0285f9dde.control:head [watch ping cookie 94849229681152] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e97) 2024-04-07T14:50:09.458 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:50:09 smithi189 podman[57239]: 2024-04-07 14:50:09.255884913 +0000 UTC m=+1.033627921 container init 0d73f39d792d88723a6b4d61f26ac9a2ca6ee1e19eb441f021b2aa90e97631fc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6-deactivate, release=754, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., version=8, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, distribution-scope=public, io.openshift.expose-services=, io.buildah.version=1.19.8, RELEASE=HEAD, ceph=True, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-04-07T14:50:09.458 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:50:09 smithi189 podman[57239]: 2024-04-07 14:50:09.259511587 +0000 UTC m=+1.037254596 container start 0d73f39d792d88723a6b4d61f26ac9a2ca6ee1e19eb441f021b2aa90e97631fc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6-deactivate, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, GIT_CLEAN=True, com.redhat.component=centos-stream-container, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, RELEASE=HEAD, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, io.buildah.version=1.19.8) 2024-04-07T14:50:09.458 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:50:09 smithi189 podman[57239]: 2024-04-07 14:50:09.35541464 +0000 UTC m=+1.133157645 container attach 0d73f39d792d88723a6b4d61f26ac9a2ca6ee1e19eb441f021b2aa90e97631fc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6-deactivate, GIT_BRANCH=HEAD, name=centos-stream, com.redhat.component=centos-stream-container, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , architecture=x86_64, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, release=754) 2024-04-07T14:50:09.831 INFO:journalctl@ceph.osd.6.smithi189.stdout:Apr 07 14:50:09 smithi189 podman[57239]: 2024-04-07 14:50:09.5403829 +0000 UTC m=+1.318125895 container died 0d73f39d792d88723a6b4d61f26ac9a2ca6ee1e19eb441f021b2aa90e97631fc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-6-deactivate, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, io.buildah.version=1.19.8, name=centos-stream, vendor=Red Hat, Inc., ceph=True, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, release=754, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, version=8, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=) 2024-04-07T14:50:09.832 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:09 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:09.781+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:12.776374+0000 front 2024-04-07T14:49:12.776258+0000 (oldest deadline 2024-04-07T14:49:44.576770+0000) 2024-04-07T14:50:09.832 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:09 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:09.781+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:24.577152+0000 front 2024-04-07T14:49:24.577247+0000 (oldest deadline 2024-04-07T14:49:50.477195+0000) 2024-04-07T14:50:09.832 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:09 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:09.781+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-07T14:49:35.678711+0000 front 2024-04-07T14:49:35.678735+0000 (oldest deadline 2024-04-07T14:50:00.378605+0000) 2024-04-07T14:50:09.832 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:09 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:09.781+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-04-07T14:49:40.379331+0000 front 2024-04-07T14:49:40.379277+0000 (oldest deadline 2024-04-07T14:50:04.479122+0000) 2024-04-07T14:50:09.832 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:09 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:09.781+0000 7f7c963ef700 -1 osd.7 97 get_health_metrics reporting 5 slow ops, oldest is osd_op(client.24637.0:128 2.0 2:02bc1a72:::realms.3a89d6cc-37c5-47ad-85a8-f8c0285f9dde.control:head [watch ping cookie 94849229681152] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e97) 2024-04-07T14:50:10.105 DEBUG:teuthology.orchestra.run.smithi189:> sudo pkill -f 'journalctl -f -n 0 -u ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@osd.6.service' 2024-04-07T14:50:10.146 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-07T14:50:10.147 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-04-07T14:50:10.147 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-04-07T14:50:10.147 DEBUG:teuthology.orchestra.run.smithi189:> sudo systemctl stop ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@osd.7 2024-04-07T14:50:10.393 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:10 smithi189 systemd[1]: Stopping Ceph osd.7 for 7388281a-f4eb-11ee-b649-cb9ed24678a4... 2024-04-07T14:50:10.393 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:10 smithi189 bash[57402]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd.7" found: no such container 2024-04-07T14:50:10.707 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:10 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:10.394+0000 7f7c98a52700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-07T14:50:10.708 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:10 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:10.394+0000 7f7c98a52700 -1 osd.7 97 *** Got signal Terminated *** 2024-04-07T14:50:10.708 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:10 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:10.394+0000 7f7c98a52700 -1 osd.7 97 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-07T14:50:11.208 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:10 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:10.829+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:12.776374+0000 front 2024-04-07T14:49:12.776258+0000 (oldest deadline 2024-04-07T14:49:44.576770+0000) 2024-04-07T14:50:11.208 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:10 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:10.829+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:24.577152+0000 front 2024-04-07T14:49:24.577247+0000 (oldest deadline 2024-04-07T14:49:50.477195+0000) 2024-04-07T14:50:11.208 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:10 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:10.829+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-07T14:49:35.678711+0000 front 2024-04-07T14:49:35.678735+0000 (oldest deadline 2024-04-07T14:50:00.378605+0000) 2024-04-07T14:50:11.208 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:10 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:10.829+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-04-07T14:49:40.379331+0000 front 2024-04-07T14:49:40.379277+0000 (oldest deadline 2024-04-07T14:50:04.479122+0000) 2024-04-07T14:50:11.208 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:10 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:10.829+0000 7f7c963ef700 -1 osd.7 97 get_health_metrics reporting 5 slow ops, oldest is osd_op(client.24637.0:128 2.0 2:02bc1a72:::realms.3a89d6cc-37c5-47ad-85a8-f8c0285f9dde.control:head [watch ping cookie 94849229681152] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e97) 2024-04-07T14:50:11.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:50:11 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:50:11.657Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": dial tcp 172.21.15.83:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:50:11.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:50:11 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:50:11.658Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": dial tcp 172.21.15.83:8443: connect: connection refused" 2024-04-07T14:50:11.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:50:11 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:50:11.658Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:50:12.207 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:11 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:11.873+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:12.776374+0000 front 2024-04-07T14:49:12.776258+0000 (oldest deadline 2024-04-07T14:49:44.576770+0000) 2024-04-07T14:50:12.207 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:11 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:11.873+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:24.577152+0000 front 2024-04-07T14:49:24.577247+0000 (oldest deadline 2024-04-07T14:49:50.477195+0000) 2024-04-07T14:50:12.208 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:11 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:11.873+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-07T14:49:35.678711+0000 front 2024-04-07T14:49:35.678735+0000 (oldest deadline 2024-04-07T14:50:00.378605+0000) 2024-04-07T14:50:12.208 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:11 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:11.873+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-04-07T14:49:40.379331+0000 front 2024-04-07T14:49:40.379277+0000 (oldest deadline 2024-04-07T14:50:04.479122+0000) 2024-04-07T14:50:12.208 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:11 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:11.873+0000 7f7c963ef700 -1 osd.7 97 get_health_metrics reporting 6 slow ops, oldest is osd_op(client.24637.0:128 2.0 2:02bc1a72:::realms.3a89d6cc-37c5-47ad-85a8-f8c0285f9dde.control:head [watch ping cookie 94849229681152] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e97) 2024-04-07T14:50:13.208 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:12 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:12.868+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:12.776374+0000 front 2024-04-07T14:49:12.776258+0000 (oldest deadline 2024-04-07T14:49:44.576770+0000) 2024-04-07T14:50:13.208 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:12 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:12.868+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:24.577152+0000 front 2024-04-07T14:49:24.577247+0000 (oldest deadline 2024-04-07T14:49:50.477195+0000) 2024-04-07T14:50:13.208 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:12 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:12.868+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-07T14:49:35.678711+0000 front 2024-04-07T14:49:35.678735+0000 (oldest deadline 2024-04-07T14:50:00.378605+0000) 2024-04-07T14:50:13.208 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:12 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:12.868+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-04-07T14:49:40.379331+0000 front 2024-04-07T14:49:40.379277+0000 (oldest deadline 2024-04-07T14:50:04.479122+0000) 2024-04-07T14:50:13.208 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:12 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:12.868+0000 7f7c963ef700 -1 osd.7 97 get_health_metrics reporting 6 slow ops, oldest is osd_op(client.24637.0:128 2.0 2:02bc1a72:::realms.3a89d6cc-37c5-47ad-85a8-f8c0285f9dde.control:head [watch ping cookie 94849229681152] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e97) 2024-04-07T14:50:13.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:50:13 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:50:13.538Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": dial tcp 172.21.15.83:8443: connect: connection refused" 2024-04-07T14:50:13.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:50:13 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:50:13.539Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": dial tcp 172.21.15.83:8443: connect: connection refused" 2024-04-07T14:50:13.925 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:50:13 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:50:13.539Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:50:14.207 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:13 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:13.893+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:12.776374+0000 front 2024-04-07T14:49:12.776258+0000 (oldest deadline 2024-04-07T14:49:44.576770+0000) 2024-04-07T14:50:14.208 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:13 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:13.893+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:24.577152+0000 front 2024-04-07T14:49:24.577247+0000 (oldest deadline 2024-04-07T14:49:50.477195+0000) 2024-04-07T14:50:14.208 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:13 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:13.893+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-07T14:49:35.678711+0000 front 2024-04-07T14:49:35.678735+0000 (oldest deadline 2024-04-07T14:50:00.378605+0000) 2024-04-07T14:50:14.208 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:13 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:13.893+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-04-07T14:49:40.379331+0000 front 2024-04-07T14:49:40.379277+0000 (oldest deadline 2024-04-07T14:50:04.479122+0000) 2024-04-07T14:50:14.208 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:13 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:13.893+0000 7f7c963ef700 -1 osd.7 97 get_health_metrics reporting 7 slow ops, oldest is osd_op(client.24637.0:128 2.0 2:02bc1a72:::realms.3a89d6cc-37c5-47ad-85a8-f8c0285f9dde.control:head [watch ping cookie 94849229681152] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e97) 2024-04-07T14:50:15.207 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:14.867+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6806 osd.0 since back 2024-04-07T14:49:12.776374+0000 front 2024-04-07T14:49:12.776258+0000 (oldest deadline 2024-04-07T14:49:44.576770+0000) 2024-04-07T14:50:15.208 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:14.867+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6814 osd.1 since back 2024-04-07T14:49:24.577152+0000 front 2024-04-07T14:49:24.577247+0000 (oldest deadline 2024-04-07T14:49:50.477195+0000) 2024-04-07T14:50:15.208 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:14.867+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6822 osd.2 since back 2024-04-07T14:49:35.678711+0000 front 2024-04-07T14:49:35.678735+0000 (oldest deadline 2024-04-07T14:50:00.378605+0000) 2024-04-07T14:50:15.208 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:14.867+0000 7f7c963ef700 -1 osd.7 97 heartbeat_check: no reply from 172.21.15.83:6830 osd.3 since back 2024-04-07T14:49:40.379331+0000 front 2024-04-07T14:49:40.379277+0000 (oldest deadline 2024-04-07T14:50:04.479122+0000) 2024-04-07T14:50:15.208 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:14 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7[39278]: 2024-04-07T14:50:14.868+0000 7f7c963ef700 -1 osd.7 97 get_health_metrics reporting 7 slow ops, oldest is osd_op(client.24637.0:128 2.0 2:02bc1a72:::realms.3a89d6cc-37c5-47ad-85a8-f8c0285f9dde.control:head [watch ping cookie 94849229681152] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e97) 2024-04-07T14:50:15.958 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:15 smithi189 podman[57414]: 2024-04-07 14:50:15.703625121 +0000 UTC m=+5.431108255 container died 8a39104a15b0cc69213ea9eef0df0a1a652cab60610a71df6e86a0f18aa06bbe (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, name=centos-stream, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., architecture=x86_64, com.redhat.component=centos-stream-container, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, vcs-type=git, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-04-07T14:50:16.343 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:16 smithi189 podman[57414]: 2024-04-07 14:50:16.127933308 +0000 UTC m=+5.855416437 container cleanup 8a39104a15b0cc69213ea9eef0df0a1a652cab60610a71df6e86a0f18aa06bbe (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7, RELEASE=HEAD, io.buildah.version=1.19.8, io.openshift.expose-services=, name=centos-stream, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, release=754) 2024-04-07T14:50:16.343 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:16 smithi189 bash[57414]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7 2024-04-07T14:50:16.344 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:16 smithi189 bash[57446]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd.7" found: no such container 2024-04-07T14:50:16.666 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:16 smithi189 podman[57432]: 2024-04-07 14:50:16.345850858 +0000 UTC m=+0.640967669 container remove 8a39104a15b0cc69213ea9eef0df0a1a652cab60610a71df6e86a0f18aa06bbe (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, RELEASE=HEAD, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, distribution-scope=public, version=8, GIT_BRANCH=HEAD, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=) 2024-04-07T14:50:16.667 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:16 smithi189 podman[57509]: 2024-04-07 14:50:16.563398303 +0000 UTC m=+0.108761325 container create b702d8d32046117617c7300f9cb74e03b21ab3f2ac871622fa324bbfca7943c8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7-deactivate, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, architecture=x86_64, GIT_BRANCH=HEAD, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , vcs-type=git, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public) 2024-04-07T14:50:16.667 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:16 smithi189 podman[57509]: 2024-04-07 14:50:16.473643499 +0000 UTC m=+0.019006529 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-07T14:50:17.660 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:17 smithi189 podman[57509]: 2024-04-07 14:50:17.394651057 +0000 UTC m=+0.940014079 container init b702d8d32046117617c7300f9cb74e03b21ab3f2ac871622fa324bbfca7943c8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7-deactivate, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , vcs-type=git, ceph=True, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=) 2024-04-07T14:50:17.660 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:17 smithi189 podman[57509]: 2024-04-07 14:50:17.397902354 +0000 UTC m=+0.943265378 container start b702d8d32046117617c7300f9cb74e03b21ab3f2ac871622fa324bbfca7943c8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7-deactivate, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, GIT_CLEAN=True, RELEASE=HEAD, name=centos-stream, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, vcs-type=git, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0) 2024-04-07T14:50:17.660 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:17 smithi189 podman[57509]: 2024-04-07 14:50:17.478096252 +0000 UTC m=+1.023459288 container attach b702d8d32046117617c7300f9cb74e03b21ab3f2ac871622fa324bbfca7943c8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7-deactivate, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, GIT_BRANCH=HEAD, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, ceph=True) 2024-04-07T14:50:18.189 INFO:journalctl@ceph.osd.7.smithi189.stdout:Apr 07 14:50:17 smithi189 podman[57509]: 2024-04-07 14:50:17.932097913 +0000 UTC m=+1.477460955 container died b702d8d32046117617c7300f9cb74e03b21ab3f2ac871622fa324bbfca7943c8 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-osd-7-deactivate, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, GIT_BRANCH=HEAD, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-04-07T14:50:18.464 DEBUG:teuthology.orchestra.run.smithi189:> sudo pkill -f 'journalctl -f -n 0 -u ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@osd.7.service' 2024-04-07T14:50:18.507 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-07T14:50:18.507 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-04-07T14:50:18.507 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-04-07T14:50:18.508 DEBUG:teuthology.orchestra.run.smithi189:> sudo systemctl stop ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@prometheus.a 2024-04-07T14:50:18.708 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:50:18 smithi189 systemd[1]: Stopping Ceph prometheus.a for 7388281a-f4eb-11ee-b649-cb9ed24678a4... 2024-04-07T14:50:18.708 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:50:18 smithi189 bash[57628]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus.a" found: no such container 2024-04-07T14:50:18.994 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:50:18 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[45663]: ts=2024-04-07T14:50:18.711Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-07T14:50:18.994 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:50:18 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[45663]: ts=2024-04-07T14:50:18.712Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-07T14:50:18.994 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:50:18 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[45663]: ts=2024-04-07T14:50:18.712Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-07T14:50:18.995 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:50:18 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[45663]: ts=2024-04-07T14:50:18.712Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-07T14:50:18.995 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:50:18 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[45663]: ts=2024-04-07T14:50:18.712Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-07T14:50:18.995 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:50:18 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[45663]: ts=2024-04-07T14:50:18.712Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-07T14:50:18.995 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:50:18 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[45663]: ts=2024-04-07T14:50:18.712Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-07T14:50:18.995 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:50:18 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[45663]: ts=2024-04-07T14:50:18.712Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-07T14:50:18.995 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:50:18 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[45663]: ts=2024-04-07T14:50:18.712Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-07T14:50:18.995 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:50:18 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[45663]: ts=2024-04-07T14:50:18.712Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-07T14:50:18.995 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:50:18 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[45663]: ts=2024-04-07T14:50:18.712Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-07T14:50:18.996 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:50:18 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a[45663]: ts=2024-04-07T14:50:18.712Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-07T14:50:18.996 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:50:18 smithi189 podman[57642]: 2024-04-07 14:50:18.722423302 +0000 UTC m=+0.100482652 container died 1aeb972809bddb06ab7df776a72446e3bd027bcaee8bab6eb46a6d503953b91e (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-07T14:50:19.335 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:50:19 smithi189 podman[57642]: 2024-04-07 14:50:19.103358749 +0000 UTC m=+0.481418141 container cleanup 1aeb972809bddb06ab7df776a72446e3bd027bcaee8bab6eb46a6d503953b91e (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-07T14:50:19.335 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:50:19 smithi189 bash[57642]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus-a 2024-04-07T14:50:19.335 INFO:journalctl@ceph.prometheus.a.smithi189.stdout:Apr 07 14:50:19 smithi189 bash[57668]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-prometheus.a" found: no such container 2024-04-07T14:50:19.340 DEBUG:teuthology.orchestra.run.smithi189:> sudo pkill -f 'journalctl -f -n 0 -u ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@prometheus.a.service' 2024-04-07T14:50:19.376 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-07T14:50:19.377 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-04-07T14:50:19.377 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 --force --keep-logs 2024-04-07T14:50:21.236 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Apr 07 14:50:21 smithi083 systemd[1]: Stopping Ceph node-exporter.a for 7388281a-f4eb-11ee-b649-cb9ed24678a4... 2024-04-07T14:50:21.236 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Apr 07 14:50:21 smithi083 bash[68402]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter.a" found: no such container 2024-04-07T14:50:21.492 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Apr 07 14:50:21 smithi083 podman[68413]: 2024-04-07 14:50:21.238233649 +0000 UTC m=+0.114631100 container died a8c27029d23c28d14db040d628deab5a8dcd1184760a15b0c9561adfbdbe511c (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-07T14:50:21.851 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Apr 07 14:50:21 smithi083 podman[68413]: 2024-04-07 14:50:21.677999005 +0000 UTC m=+0.554396488 container cleanup a8c27029d23c28d14db040d628deab5a8dcd1184760a15b0c9561adfbdbe511c (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-07T14:50:21.851 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Apr 07 14:50:21 smithi083 bash[68413]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-a 2024-04-07T14:50:21.852 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:50:21 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=error ts=2024-04-07T14:50:21.658Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.83:8443//api/prometheus_receiver\": dial tcp 172.21.15.83:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:50:21.852 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:50:21 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:50:21.659Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.83:8443//api/prometheus_receiver\": dial tcp 172.21.15.83:8443: connect: connection refused" 2024-04-07T14:50:21.852 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:50:21 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=warn ts=2024-04-07T14:50:21.659Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.189:8443/api/prometheus_receiver\": dial tcp 172.21.15.189:8443: connect: connection refused" 2024-04-07T14:50:22.175 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:50:22 smithi083 systemd[1]: Stopping Ceph alertmanager.a for 7388281a-f4eb-11ee-b649-cb9ed24678a4... 2024-04-07T14:50:22.175 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:50:22 smithi083 bash[68492]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager.a" found: no such container 2024-04-07T14:50:22.175 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Apr 07 14:50:21 smithi083 podman[68427]: 2024-04-07 14:50:21.854069498 +0000 UTC m=+0.614425554 container remove a8c27029d23c28d14db040d628deab5a8dcd1184760a15b0c9561adfbdbe511c (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-07T14:50:22.175 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Apr 07 14:50:21 smithi083 systemd[1]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-07T14:50:22.175 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Apr 07 14:50:21 smithi083 bash[68439]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter.a" found: no such container 2024-04-07T14:50:22.176 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Apr 07 14:50:21 smithi083 systemd[1]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-07T14:50:22.176 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Apr 07 14:50:21 smithi083 systemd[1]: Stopped Ceph node-exporter.a for 7388281a-f4eb-11ee-b649-cb9ed24678a4. 2024-04-07T14:50:22.176 INFO:journalctl@ceph.node-exporter.a.smithi083.stdout:Apr 07 14:50:21 smithi083 systemd[1]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@node-exporter.a.service: Consumed 5.500s CPU time. 2024-04-07T14:50:22.472 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:50:22 smithi083 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a[51662]: level=info ts=2024-04-07T14:50:22.207Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-04-07T14:50:22.472 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:50:22 smithi083 podman[68505]: 2024-04-07 14:50:22.227957676 +0000 UTC m=+0.121492641 container died 9f4fc57a8aba2c3ec0d31b6a658b17fc81706051a1eac5d62baf9cad2b90d2f5 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-07T14:50:22.856 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:50:22 smithi083 podman[68505]: 2024-04-07 14:50:22.590844341 +0000 UTC m=+0.484379311 container cleanup 9f4fc57a8aba2c3ec0d31b6a658b17fc81706051a1eac5d62baf9cad2b90d2f5 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-07T14:50:22.856 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:50:22 smithi083 bash[68505]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a 2024-04-07T14:50:22.856 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:50:22 smithi083 bash[68532]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager.a" found: no such container 2024-04-07T14:50:23.110 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:50:22 smithi083 podman[68518]: 2024-04-07 14:50:22.858789347 +0000 UTC m=+0.629489601 container remove 9f4fc57a8aba2c3ec0d31b6a658b17fc81706051a1eac5d62baf9cad2b90d2f5 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-07T14:50:23.111 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:50:22 smithi083 podman[68518]: 2024-04-07 14:50:22.934314433 +0000 UTC m=+0.705014645 volume remove 3a3e8a1591b73c5e964af041dd6baaf7e540f333da740d55ef0b104d4afb0296 2024-04-07T14:50:23.111 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:50:22 smithi083 systemd[1]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@alertmanager.a.service: Deactivated successfully. 2024-04-07T14:50:23.111 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:50:22 smithi083 systemd[1]: Stopped Ceph alertmanager.a for 7388281a-f4eb-11ee-b649-cb9ed24678a4. 2024-04-07T14:50:23.111 INFO:journalctl@ceph.alertmanager.a.smithi083.stdout:Apr 07 14:50:22 smithi083 systemd[1]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@alertmanager.a.service: Consumed 1.887s CPU time. 2024-04-07T14:50:45.956 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 --force --keep-logs 2024-04-07T14:50:47.688 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:50:47 smithi189 systemd[1]: Stopping Ceph node-exporter.b for 7388281a-f4eb-11ee-b649-cb9ed24678a4... 2024-04-07T14:50:47.688 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:50:47 smithi189 bash[58681]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter.b" found: no such container 2024-04-07T14:50:47.953 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:50:47 smithi189 podman[58694]: 2024-04-07 14:50:47.690449415 +0000 UTC m=+0.083188677 container died f7de611efab077414f319cd654b3c6f17fa87db6c7e3f89e7dbccf61da32b043 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-07T14:50:48.207 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:50:48 smithi189 podman[58694]: 2024-04-07 14:50:48.05480634 +0000 UTC m=+0.447545605 container cleanup f7de611efab077414f319cd654b3c6f17fa87db6c7e3f89e7dbccf61da32b043 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-07T14:50:48.207 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:50:48 smithi189 bash[58694]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b 2024-04-07T14:50:48.208 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:50:48 smithi189 bash[58720]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter.b" found: no such container 2024-04-07T14:50:48.640 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:50:48 smithi189 systemd[1]: Stopping Ceph grafana.a for 7388281a-f4eb-11ee-b649-cb9ed24678a4... 2024-04-07T14:50:48.641 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:50:48 smithi189 bash[58773]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana.a" found: no such container 2024-04-07T14:50:48.641 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:50:48 smithi189 podman[58708]: 2024-04-07 14:50:48.322727249 +0000 UTC m=+0.633072636 container remove f7de611efab077414f319cd654b3c6f17fa87db6c7e3f89e7dbccf61da32b043 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-07T14:50:48.641 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:50:48 smithi189 systemd[1]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-07T14:50:48.641 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:50:48 smithi189 systemd[1]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-07T14:50:48.641 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:50:48 smithi189 systemd[1]: Stopped Ceph node-exporter.b for 7388281a-f4eb-11ee-b649-cb9ed24678a4. 2024-04-07T14:50:48.641 INFO:journalctl@ceph.node-exporter.b.smithi189.stdout:Apr 07 14:50:48 smithi189 systemd[1]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@node-exporter.b.service: Consumed 5.280s CPU time. 2024-04-07T14:50:48.907 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:50:48 smithi189 ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a[44650]: t=2024-04-07T14:50:48+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-04-07T14:50:48.907 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:50:48 smithi189 podman[58786]: 2024-04-07 14:50:48.653186702 +0000 UTC m=+0.118126964 container died cea72126e7bd2b90832ad688c38a06eeadad601787b38a858219cc97177171ff (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, vendor=Red Hat, Inc., vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, io.buildah.version=1.24.2, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, architecture=x86_64, distribution-scope=public, version=8.5, build-date=2022-03-28T10:36:18.413762, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=ubi8-container, release=236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-04-07T14:50:49.157 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:50:48 smithi189 podman[58786]: 2024-04-07 14:50:48.992096062 +0000 UTC m=+0.457036331 container cleanup cea72126e7bd2b90832ad688c38a06eeadad601787b38a858219cc97177171ff (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a, name=ubi8, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, build-date=2022-03-28T10:36:18.413762, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.24.2, vendor=Red Hat, Inc., release=236.1648460182, vcs-type=git, architecture=x86_64, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-07T14:50:49.157 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:50:48 smithi189 bash[58786]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a 2024-04-07T14:50:49.458 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:50:49 smithi189 podman[58800]: 2024-04-07 14:50:49.15993648 +0000 UTC m=+0.505801016 container remove cea72126e7bd2b90832ad688c38a06eeadad601787b38a858219cc97177171ff (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana-a, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-type=git, description=Ceph Grafana Container, vendor=Red Hat, Inc., io.openshift.expose-services=, io.openshift.tags=base rhel8, build-date=2022-03-28T10:36:18.413762, maintainer=Paul Cuzner , release=236.1648460182, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.24.2, version=8.5) 2024-04-07T14:50:49.458 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:50:49 smithi189 bash[58812]: Error: no container with name or ID "ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4-grafana.a" found: no such container 2024-04-07T14:50:49.458 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:50:49 smithi189 systemd[1]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@grafana.a.service: Deactivated successfully. 2024-04-07T14:50:49.458 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:50:49 smithi189 systemd[1]: Stopped Ceph grafana.a for 7388281a-f4eb-11ee-b649-cb9ed24678a4. 2024-04-07T14:50:49.458 INFO:journalctl@ceph.grafana.a.smithi189.stdout:Apr 07 14:50:49 smithi189 systemd[1]: ceph-7388281a-f4eb-11ee-b649-cb9ed24678a4@grafana.a.service: Consumed 4.508s CPU time. 2024-04-07T14:51:00.789 DEBUG:teuthology.orchestra.run.smithi083:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-07T14:51:00.826 DEBUG:teuthology.orchestra.run.smithi189:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-07T14:51:00.862 INFO:tasks.cephadm:Archiving crash dumps... 2024-04-07T14:51:00.864 DEBUG:teuthology.misc:Transferring archived files from smithi083:/var/lib/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/crash to /home/teuthworker/archive/teuthology-2024-04-01_20:08:13-orch-main-distro-default-smithi/7634198/remote/smithi083/crash 2024-04-07T14:51:00.865 DEBUG:teuthology.orchestra.run.smithi083:> sudo tar c -f - -C /var/lib/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/crash -- . 2024-04-07T14:51:00.915 INFO:teuthology.orchestra.run.smithi083.stderr:tar: /var/lib/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/crash: Cannot open: No such file or directory 2024-04-07T14:51:00.916 INFO:teuthology.orchestra.run.smithi083.stderr:tar: Error is not recoverable: exiting now 2024-04-07T14:51:00.917 DEBUG:teuthology.misc:Transferring archived files from smithi189:/var/lib/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/crash to /home/teuthworker/archive/teuthology-2024-04-01_20:08:13-orch-main-distro-default-smithi/7634198/remote/smithi189/crash 2024-04-07T14:51:00.918 DEBUG:teuthology.orchestra.run.smithi189:> sudo tar c -f - -C /var/lib/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/crash -- . 2024-04-07T14:51:00.988 INFO:teuthology.orchestra.run.smithi189.stderr:tar: /var/lib/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/crash: Cannot open: No such file or directory 2024-04-07T14:51:00.988 INFO:teuthology.orchestra.run.smithi189.stderr:tar: Error is not recoverable: exiting now 2024-04-07T14:51:00.989 INFO:tasks.cephadm:Checking cluster log for badness... 2024-04-07T14:51:00.990 DEBUG:teuthology.orchestra.run.smithi083:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-07T14:51:01.021 INFO:teuthology.orchestra.run.smithi083.stdout:2024-04-07T14:33:09.885856+0000 mon.a (mon.0) 198 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-04-07T14:51:01.022 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-04-07T14:51:01.022 INFO:tasks.cephadm:Compressing logs... 2024-04-07T14:51:01.022 DEBUG:teuthology.orchestra.run.smithi083:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-07T14:51:01.065 DEBUG:teuthology.orchestra.run.smithi189:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-07T14:51:01.091 INFO:teuthology.orchestra.run.smithi083.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose --: No such file or directory /var/log/ceph/cephadm.log 2024-04-07T14:51:01.092 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-04-07T14:51:01.092 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-mon.a.log 2024-04-07T14:51:01.092 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-client.rgw.foo.smithi083.oslioj.log 2024-04-07T14:51:01.092 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph.audit.log 2024-04-07T14:51:01.096 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-osd.1.log 2024-04-07T14:51:01.096 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-mon.a.log: /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-client.rgw.foo.smithi083.oslioj.log: /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph.audit.log: 62.5% -- replaced with /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-client.rgw.foo.smithi083.oslioj.log.gz 2024-04-07T14:51:01.097 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-osd.3.log 2024-04-07T14:51:01.097 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-mgr.y.log 2024-04-07T14:51:01.097 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-osd.1.log: /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/tcmu-runner.log 2024-04-07T14:51:01.097 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-osd.0.log 2024-04-07T14:51:01.097 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/tcmu-runner.log: gzip -5 --verbose 66.6% -- replaced with /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/tcmu-runner.log.gz 2024-04-07T14:51:01.097 INFO:teuthology.orchestra.run.smithi083.stderr: -- /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph.cephadm.log 2024-04-07T14:51:01.097 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph.log 2024-04-07T14:51:01.097 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph.cephadm.log: /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-osd.0.log: 89.5% -- replaced with /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph.cephadm.log.gz 2024-04-07T14:51:01.097 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-osd.2.log 2024-04-07T14:51:01.101 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph.log: 94.1% -- replaced with /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph.audit.log.gz 2024-04-07T14:51:01.102 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-volume.log 2024-04-07T14:51:01.102 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-osd.2.log: 93.5% -- replaced with /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph.log.gz 2024-04-07T14:51:01.102 INFO:teuthology.orchestra.run.smithi083.stderr:gzip -5 --verbose -- /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-mon.c.log 2024-04-07T14:51:01.103 INFO:teuthology.orchestra.run.smithi189.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-07T14:51:01.103 INFO:teuthology.orchestra.run.smithi189.stderr:: No such file or directory 2024-04-07T14:51:01.103 INFO:teuthology.orchestra.run.smithi189.stderr:gzip -5 --verbose -- /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-osd.6.log 2024-04-07T14:51:01.103 INFO:teuthology.orchestra.run.smithi189.stderr:gzip -5 --verbose -- /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-client.rgw.foo.smithi189.taxdrv.log 2024-04-07T14:51:01.103 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/cephadm.log: /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph.audit.log 2024-04-07T14:51:01.103 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-client.rgw.foo.smithi189.taxdrv.log: gzip -5 --verbose -- /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-osd.4.log 2024-04-07T14:51:01.104 INFO:teuthology.orchestra.run.smithi189.stderr: 62.2% -- replaced with /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-client.rgw.foo.smithi189.taxdrv.log.gz 2024-04-07T14:51:01.104 INFO:teuthology.orchestra.run.smithi189.stderr:gzip -5 --verbose -- /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-osd.7.log 2024-04-07T14:51:01.104 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph.audit.log: /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-osd.5.log 2024-04-07T14:51:01.104 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-mgr.x.log 2024-04-07T14:51:01.104 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph.cephadm.log 2024-04-07T14:51:01.104 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-mgr.x.log: 90.6% -- replaced with /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph.audit.log.gz 2024-04-07T14:51:01.109 INFO:teuthology.orchestra.run.smithi189.stderr:gzip -5 --verbose -- /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph.log 2024-04-07T14:51:01.109 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph.cephadm.log: 82.0% -- replaced with /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph.cephadm.log.gz 2024-04-07T14:51:01.114 INFO:teuthology.orchestra.run.smithi189.stderr: 92.2% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-07T14:51:01.116 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-volume.log: 92.4% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-07T14:51:01.121 INFO:teuthology.orchestra.run.smithi189.stderr:gzip -5 --verbose -- /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-volume.log 2024-04-07T14:51:01.122 INFO:teuthology.orchestra.run.smithi189.stderr:gzip/var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph.log: -5 --verbose -- /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-mon.b.log 2024-04-07T14:51:01.123 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-volume.log: 88.4% -- replaced with /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph.log.gz 2024-04-07T14:51:01.124 INFO:teuthology.orchestra.run.smithi189.stderr: 92.5% -- replaced with /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-mgr.x.log.gz 2024-04-07T14:51:01.132 INFO:teuthology.orchestra.run.smithi083.stderr:/var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-mon.c.log: 92.5% -- replaced with /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-volume.log.gz 2024-04-07T14:51:01.147 INFO:teuthology.orchestra.run.smithi189.stderr:/var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-mon.b.log: 92.6% -- replaced with /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-volume.log.gz 2024-04-07T14:51:01.327 INFO:teuthology.orchestra.run.smithi083.stderr: 87.4% -- replaced with /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-mgr.y.log.gz 2024-04-07T14:51:01.386 INFO:teuthology.orchestra.run.smithi083.stderr: 92.6% -- replaced with /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-mon.c.log.gz 2024-04-07T14:51:01.395 INFO:teuthology.orchestra.run.smithi189.stderr: 92.4% -- replaced with /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-mon.b.log.gz 2024-04-07T14:51:01.478 INFO:teuthology.orchestra.run.smithi083.stderr: 93.1% -- replaced with /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-osd.2.log.gz 2024-04-07T14:51:01.597 INFO:teuthology.orchestra.run.smithi189.stderr: 93.1% -- replaced with /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-osd.6.log.gz 2024-04-07T14:51:01.720 INFO:teuthology.orchestra.run.smithi083.stderr: 93.3% -- replaced with /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-osd.0.log.gz 2024-04-07T14:51:01.749 INFO:teuthology.orchestra.run.smithi189.stderr: 93.4% -- replaced with /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-osd.4.log.gz 2024-04-07T14:51:01.765 INFO:teuthology.orchestra.run.smithi189.stderr: 93.4% -- replaced with /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-osd.7.log.gz 2024-04-07T14:51:01.768 INFO:teuthology.orchestra.run.smithi189.stderr: 93.2% -- replaced with /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-osd.5.log.gz 2024-04-07T14:51:01.770 INFO:teuthology.orchestra.run.smithi189.stderr: 2024-04-07T14:51:01.770 INFO:teuthology.orchestra.run.smithi189.stderr:real 0m0.690s 2024-04-07T14:51:01.770 INFO:teuthology.orchestra.run.smithi189.stderr:user 0m2.740s 2024-04-07T14:51:01.770 INFO:teuthology.orchestra.run.smithi189.stderr:sys 0m0.138s 2024-04-07T14:51:01.824 INFO:teuthology.orchestra.run.smithi083.stderr: 93.3% -- replaced with /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-osd.1.log.gz 2024-04-07T14:51:01.829 INFO:teuthology.orchestra.run.smithi083.stderr: 91.1% -- replaced with /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-mon.a.log.gz 2024-04-07T14:51:01.845 INFO:teuthology.orchestra.run.smithi083.stderr: 93.3% -- replaced with /var/log/ceph/7388281a-f4eb-11ee-b649-cb9ed24678a4/ceph-osd.3.log.gz 2024-04-07T14:51:01.848 INFO:teuthology.orchestra.run.smithi083.stderr: 2024-04-07T14:51:01.848 INFO:teuthology.orchestra.run.smithi083.stderr:real 0m0.770s 2024-04-07T14:51:01.848 INFO:teuthology.orchestra.run.smithi083.stderr:user 0m3.663s 2024-04-07T14:51:01.848 INFO:teuthology.orchestra.run.smithi083.stderr:sys 0m0.159s 2024-04-07T14:51:01.848 INFO:tasks.cephadm:Archiving logs... 2024-04-07T14:51:01.849 DEBUG:teuthology.misc:Transferring archived files from smithi083:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-01_20:08:13-orch-main-distro-default-smithi/7634198/remote/smithi083/log 2024-04-07T14:51:01.850 DEBUG:teuthology.orchestra.run.smithi083:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-07T14:51:02.403 DEBUG:teuthology.misc:Transferring archived files from smithi189:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-01_20:08:13-orch-main-distro-default-smithi/7634198/remote/smithi189/log 2024-04-07T14:51:02.404 DEBUG:teuthology.orchestra.run.smithi189:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-07T14:51:02.797 INFO:tasks.cephadm:Removing cluster... 2024-04-07T14:51:02.798 DEBUG:teuthology.orchestra.run.smithi083:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 --force 2024-04-07T14:51:03.074 DEBUG:teuthology.orchestra.run.smithi189:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 --force 2024-04-07T14:51:03.351 INFO:tasks.cephadm:Removing cephadm ... 2024-04-07T14:51:03.351 DEBUG:teuthology.orchestra.run.smithi083:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-07T14:51:03.366 DEBUG:teuthology.orchestra.run.smithi189:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-07T14:51:03.382 INFO:tasks.cephadm:Teardown complete 2024-04-07T14:51:03.383 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-04-07T14:51:03.394 INFO:teuthology.task.clock:Checking final clock skew... 2024-04-07T14:51:03.394 DEBUG:teuthology.orchestra.run.smithi083:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-07T14:51:03.409 DEBUG:teuthology.orchestra.run.smithi189:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-07T14:51:03.420 INFO:teuthology.orchestra.run.smithi083.stderr:bash: line 1: ntpq: command not found 2024-04-07T14:51:03.426 INFO:teuthology.orchestra.run.smithi083.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-07T14:51:03.426 INFO:teuthology.orchestra.run.smithi083.stdout:=============================================================================== 2024-04-07T14:51:03.426 INFO:teuthology.orchestra.run.smithi083.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 34 -672us[ -674us] +/- 83ms 2024-04-07T14:51:03.426 INFO:teuthology.orchestra.run.smithi083.stdout:^* hv02.front.sepia.ceph.com 3 6 377 34 +52us[ +50us] +/- 55ms 2024-04-07T14:51:03.427 INFO:teuthology.orchestra.run.smithi083.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 35 +529us[ +527us] +/- 68ms 2024-04-07T14:51:03.427 INFO:teuthology.orchestra.run.smithi083.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-07T14:51:03.435 INFO:teuthology.orchestra.run.smithi189.stderr:bash: line 1: ntpq: command not found 2024-04-07T14:51:03.440 INFO:teuthology.orchestra.run.smithi189.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-07T14:51:03.441 INFO:teuthology.orchestra.run.smithi189.stdout:=============================================================================== 2024-04-07T14:51:03.441 INFO:teuthology.orchestra.run.smithi189.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 35 -673us[ -673us] +/- 83ms 2024-04-07T14:51:03.441 INFO:teuthology.orchestra.run.smithi189.stdout:^* hv02.front.sepia.ceph.com 3 6 377 38 +41us[ +62us] +/- 54ms 2024-04-07T14:51:03.441 INFO:teuthology.orchestra.run.smithi189.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 36 +516us[ +516us] +/- 68ms 2024-04-07T14:51:03.441 INFO:teuthology.orchestra.run.smithi189.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-07T14:51:03.442 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-07T14:51:03.451 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-07T14:51:03.453 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-07T14:51:03.476 DEBUG:teuthology.orchestra.run.smithi083:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-07T14:51:03.550 DEBUG:teuthology.orchestra.run.smithi189:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-07T14:51:03.622 DEBUG:teuthology.orchestra.run.smithi083:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-07T14:51:03.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-07T14:51:03.729 DEBUG:teuthology.orchestra.run.smithi189:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-07T14:51:03.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-07T14:51:03.758 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-07T14:51:03.773 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-07T14:51:03.883 INFO:teuthology.task.internal:Duration was 1474.998031 seconds 2024-04-07T14:51:03.884 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-07T14:51:03.978 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-07T14:51:03.979 DEBUG:teuthology.orchestra.run.smithi083:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-07T14:51:03.981 DEBUG:teuthology.orchestra.run.smithi189:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-07T14:51:04.021 INFO:teuthology.orchestra.run.smithi083.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-07T14:51:04.026 INFO:teuthology.orchestra.run.smithi189.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-07T14:51:04.498 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-07T14:51:04.498 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi083.front.sepia.ceph.com 2024-04-07T14:51:04.499 DEBUG:teuthology.orchestra.run.smithi083:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-07T14:51:04.522 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi189.front.sepia.ceph.com 2024-04-07T14:51:04.522 DEBUG:teuthology.orchestra.run.smithi189:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-07T14:51:04.548 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-07T14:51:04.548 DEBUG:teuthology.orchestra.run.smithi083:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-07T14:51:04.565 DEBUG:teuthology.orchestra.run.smithi189:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-07T14:51:04.618 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-07T14:51:04.619 DEBUG:teuthology.orchestra.run.smithi083:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-07T14:51:04.634 DEBUG:teuthology.orchestra.run.smithi189:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-07T14:51:04.990 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-07T14:51:05.001 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-07T14:51:05.001 DEBUG:teuthology.orchestra.run.smithi083:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-07T14:51:05.028 DEBUG:teuthology.orchestra.run.smithi189:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-07T14:51:05.056 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-07T14:51:05.066 DEBUG:teuthology.orchestra.run.smithi083:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-07T14:51:05.071 DEBUG:teuthology.orchestra.run.smithi189:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-07T14:51:05.093 INFO:teuthology.orchestra.run.smithi083.stdout:kernel.core_pattern = core 2024-04-07T14:51:05.122 INFO:teuthology.orchestra.run.smithi189.stdout:kernel.core_pattern = core 2024-04-07T14:51:05.144 DEBUG:teuthology.orchestra.run.smithi083:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-07T14:51:05.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-07T14:51:05.180 DEBUG:teuthology.orchestra.run.smithi189:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-07T14:51:05.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-07T14:51:05.199 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-07T14:51:05.209 INFO:teuthology.task.internal:Transferring archived files... 2024-04-07T14:51:05.210 DEBUG:teuthology.misc:Transferring archived files from smithi083:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-01_20:08:13-orch-main-distro-default-smithi/7634198/remote/smithi083 2024-04-07T14:51:05.210 DEBUG:teuthology.orchestra.run.smithi083:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-07T14:51:05.269 DEBUG:teuthology.misc:Transferring archived files from smithi189:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-01_20:08:13-orch-main-distro-default-smithi/7634198/remote/smithi189 2024-04-07T14:51:05.269 DEBUG:teuthology.orchestra.run.smithi189:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-07T14:51:05.311 INFO:teuthology.task.internal:Removing archive directory... 2024-04-07T14:51:05.312 DEBUG:teuthology.orchestra.run.smithi083:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-07T14:51:05.314 DEBUG:teuthology.orchestra.run.smithi189:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-07T14:51:05.359 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-07T14:51:05.370 INFO:teuthology.task.internal:Not uploading archives. 2024-04-07T14:51:05.371 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-07T14:51:05.404 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-07T14:51:05.404 DEBUG:teuthology.orchestra.run.smithi083:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-07T14:51:05.407 DEBUG:teuthology.orchestra.run.smithi189:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-07T14:51:05.419 INFO:teuthology.orchestra.run.smithi083.stdout: 266043 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 7 14:51 /home/ubuntu/cephtest 2024-04-07T14:51:05.421 INFO:teuthology.orchestra.run.smithi189.stdout: 262166 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 7 14:51 /home/ubuntu/cephtest 2024-04-07T14:51:05.422 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-07T14:51:05.432 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-07T14:51:05.546 INFO:teuthology.nuke:Checking targets against current locks 2024-04-07T14:51:05.577 DEBUG:teuthology.nuke:shortname: smithi083 2024-04-07T14:51:05.577 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-07T14:51:05.600 DEBUG:teuthology.nuke:shortname: smithi189 2024-04-07T14:51:05.600 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-07T14:51:05.604 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi083.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-01_20:08:13-orch-main-distro-default-smithi/7634198', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-07 14:17:18.353314', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAfjhHyLOYAGOtB1YzxEgzc07ihb/9sedUE5obRqGNqEj3nlEEgCNd9gfUZYMGlvZzmGXYoun+d6e7CX6G6X4mk='} 2024-04-07T14:51:05.627 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi189.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-01_20:08:13-orch-main-distro-default-smithi/7634198', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-07 14:17:18.351773', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKjMxsrnDBjL2m/RVeBcTmi2M6kn6U6nmOsSBocU6Y+64PjMMqutRuS9mxF5xWLknvGcAlMMyjpNkUrfVBhuFH0='} 2024-04-07T14:51:05.657 INFO:teuthology.orchestra.console.smithi083:Power off 2024-04-07T14:51:05.657 DEBUG:teuthology.orchestra.console.smithi083:pexpect command: ipmitool -H smithi083.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-07T14:51:05.673 INFO:teuthology.orchestra.console.smithi189:Power off 2024-04-07T14:51:05.673 DEBUG:teuthology.orchestra.console.smithi189:pexpect command: ipmitool -H smithi189.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-07T14:51:05.682 DEBUG:teuthology.orchestra.console.smithi083:power off output: Chassis Power Control: Down/Off 2024-04-07T14:51:05.683 DEBUG:teuthology.orchestra.console.smithi083:pexpect command: ipmitool -H smithi083.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-07T14:51:05.696 DEBUG:teuthology.orchestra.console.smithi189:power off output: Chassis Power Control: Down/Off 2024-04-07T14:51:05.696 DEBUG:teuthology.orchestra.console.smithi189:pexpect command: ipmitool -H smithi189.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-07T14:51:05.704 DEBUG:teuthology.orchestra.console.smithi083:check power output: Chassis Power is on 2024-04-07T14:51:05.717 DEBUG:teuthology.orchestra.console.smithi189:check power output: Chassis Power is on 2024-04-07T14:51:09.705 DEBUG:teuthology.orchestra.console.smithi083:pexpect command: ipmitool -H smithi083.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-07T14:51:09.718 DEBUG:teuthology.orchestra.console.smithi189:pexpect command: ipmitool -H smithi189.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-07T14:51:09.818 DEBUG:teuthology.orchestra.console.smithi083:check power output: Chassis Power is on 2024-04-07T14:51:09.830 DEBUG:teuthology.orchestra.console.smithi189:check power output: Chassis Power is on 2024-04-07T14:51:13.823 DEBUG:teuthology.orchestra.console.smithi083:pexpect command: ipmitool -H smithi083.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-07T14:51:13.834 DEBUG:teuthology.orchestra.console.smithi189:pexpect command: ipmitool -H smithi189.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-07T14:51:13.937 DEBUG:teuthology.orchestra.console.smithi083:check power output: Chassis Power is on 2024-04-07T14:51:13.945 DEBUG:teuthology.orchestra.console.smithi189:check power output: Chassis Power is off 2024-04-07T14:51:14.046 INFO:teuthology.orchestra.console.smithi189:Power off completed 2024-04-07T14:51:17.942 DEBUG:teuthology.orchestra.console.smithi083:pexpect command: ipmitool -H smithi083.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-07T14:51:18.056 DEBUG:teuthology.orchestra.console.smithi083:check power output: Chassis Power is off 2024-04-07T14:51:18.157 INFO:teuthology.orchestra.console.smithi083:Power off completed 2024-04-07T14:51:18.262 INFO:teuthology.run:Summary data: description: orch/cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/repo_digest 3-upgrade/staggered 4-wait 5-upgrade-ls agent/on mon_election/connectivity} duration: 1474.9980311393738 failure_reason: 'Command failed on smithi083 with status 22: ''sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 7388281a-f4eb-11ee-b649-cb9ed24678a4 -e sha1=e5c885fe95ee982ff7b45214daa61b1a6d09d57d -- bash -c \''ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1\''''' owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=1dd0df8d3d5f4dacbd8eaf9265a7df31 status: fail success: false 2024-04-07T14:51:18.263 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-07T14:51:18.352 INFO:teuthology.run:FAIL